reploid 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/apps/poolday/config/chat-files.json +27 -0
- package/apps/poolday/config/partition-generation.js +21 -0
- package/apps/poolday/config/partition-policy.json +74 -0
- package/apps/poolday/config/partition-releases.json +3 -0
- package/apps/poolday/config/work-profile.json +122 -0
- package/apps/poolday/host/chat-view.js +47 -0
- package/apps/poolday/host/earning/client.js +199 -0
- package/apps/poolday/host/earning/contribution.js +168 -0
- package/apps/poolday/host/partition-release.js +33 -0
- package/apps/poolday/host/work-partition-artifact-source.js +37 -0
- package/apps/poolday/host/work-partition-capsule.js +39 -0
- package/apps/poolday/host/work-partition-plan.js +167 -0
- package/apps/poolday/host/work-partition-tokenizer-guard.js +21 -0
- package/apps/poolday/infrastructure/model-storage-lock.js +22 -0
- package/apps/poolday/infrastructure/pack-release-storage.js +52 -0
- package/apps/poolday/pool/pack-operation.js +161 -0
- package/apps/poolday/pool/pack-release-policy.js +44 -0
- package/apps/poolday/pool/peer-pack-data-channel.js +217 -0
- package/apps/poolday/providers/work-adapter.js +67 -0
- package/apps/poolday/providers/work-device.js +33 -0
- package/apps/poolday/providers/work-generation-completion.js +55 -0
- package/apps/poolday/providers/work-provider.js +209 -0
- package/apps/poolday/providers/work-resident-provider.js +113 -0
- package/deploy/artifacts/doppler-gpu-0.7.1.tgz +0 -0
- package/docs/fleet-service.md +214 -0
- package/docs/native-participant.md +534 -0
- package/native-delivery.json +1174 -0
- package/package.json +29 -4
- package/packages/reploid/src/artifacts/binary.js +25 -0
- package/packages/reploid/src/artifacts/canonical-json.js +23 -0
- package/packages/reploid/src/artifacts/custody/exchange.js +402 -0
- package/packages/reploid/src/artifacts/custody/piece-acquisition.js +70 -0
- package/packages/reploid/src/artifacts/custody/runtime.js +404 -0
- package/packages/reploid/src/artifacts/custody/supply-budget.js +19 -0
- package/packages/reploid/src/artifacts/custody-certificate.js +123 -0
- package/packages/reploid/src/artifacts/delivery-attestation.js +64 -0
- package/packages/reploid/src/artifacts/generation-terminal.js +36 -0
- package/packages/reploid/src/artifacts/identity.js +91 -0
- package/packages/reploid/src/artifacts/participant-proofs.js +159 -0
- package/packages/reploid/src/artifacts/receipt.js +131 -0
- package/packages/reploid/src/artifacts/signatures.js +94 -0
- package/packages/reploid/src/artifacts/whole-model-receipt.js +42 -0
- package/packages/reploid/src/cancellation.js +44 -0
- package/packages/reploid/src/config/defaults.json +110 -0
- package/packages/reploid/src/config/index.js +209 -0
- package/packages/reploid/src/config/request-overrides.json +9 -0
- package/packages/reploid/src/config/transport-defaults.json +7 -0
- package/packages/reploid/src/hashing.js +30 -0
- package/packages/reploid/src/measurement/capacity-utilization.js +321 -0
- package/packages/reploid/src/measurement/index.js +5 -0
- package/packages/reploid/src/measurement/readiness-records.js +310 -0
- package/packages/reploid/src/mesh/computer-identity.js +50 -0
- package/packages/reploid/src/mesh/computer-identity.policy.json +8 -0
- package/packages/reploid/src/mesh/contribution.js +141 -0
- package/packages/reploid/src/mesh/generation-recovery.js +58 -0
- package/packages/reploid/src/mesh/gpu-signature-shader.js +27 -0
- package/packages/reploid/src/mesh/gpu-signature.js +136 -0
- package/packages/reploid/src/mesh/gpu-signature.policy.json +11 -0
- package/packages/reploid/src/mesh/index.js +66 -0
- package/packages/reploid/src/mesh/partitions/automatic-ordered-partitions.js +540 -0
- package/packages/reploid/src/mesh/partitions/automatic-partitions.js +298 -0
- package/packages/reploid/src/mesh/partitions/partition-attempt-evidence.js +282 -0
- package/packages/reploid/src/mesh/partitions/partition-attempt-queue-policy.json +6 -0
- package/packages/reploid/src/mesh/partitions/partition-attempt-queue.js +112 -0
- package/packages/reploid/src/mesh/partitions/partition-canonical-contract.js +167 -0
- package/packages/reploid/src/mesh/partitions/partition-canonical-execution.js +168 -0
- package/packages/reploid/src/mesh/partitions/partition-chat-canonical.js +49 -0
- package/packages/reploid/src/mesh/partitions/partition-chat.js +125 -0
- package/packages/reploid/src/mesh/partitions/partition-contract.js +192 -0
- package/packages/reploid/src/mesh/partitions/partition-device-admission.js +96 -0
- package/packages/reploid/src/mesh/partitions/partition-discovery.js +96 -0
- package/packages/reploid/src/mesh/partitions/partition-entry-publication.js +183 -0
- package/packages/reploid/src/mesh/partitions/partition-entry.js +434 -0
- package/packages/reploid/src/mesh/partitions/partition-generation-request.js +33 -0
- package/packages/reploid/src/mesh/partitions/partition-grants.js +277 -0
- package/packages/reploid/src/mesh/partitions/partition-link.js +103 -0
- package/packages/reploid/src/mesh/partitions/partition-network.js +109 -0
- package/packages/reploid/src/mesh/partitions/partition-ordered-chat.js +290 -0
- package/packages/reploid/src/mesh/partitions/partition-ordered-peer.js +415 -0
- package/packages/reploid/src/mesh/partitions/partition-ordered-publication.js +100 -0
- package/packages/reploid/src/mesh/partitions/partition-peer.js +314 -0
- package/packages/reploid/src/mesh/partitions/partition-placement.js +143 -0
- package/packages/reploid/src/mesh/partitions/partition-preparation-progress.js +25 -0
- package/packages/reploid/src/mesh/partitions/partition-receiver-policy.json +7 -0
- package/packages/reploid/src/mesh/partitions/partition-reservations.js +75 -0
- package/packages/reploid/src/mesh/partitions/partition-route-envelope-policy.json +16 -0
- package/packages/reploid/src/mesh/partitions/partition-route-envelope.js +74 -0
- package/packages/reploid/src/mesh/partitions/partition-route-execution.js +409 -0
- package/packages/reploid/src/mesh/partitions/partition-route-network.js +179 -0
- package/packages/reploid/src/mesh/partitions/partition-route-ranking.js +268 -0
- package/packages/reploid/src/mesh/partitions/partition-runner.js +319 -0
- package/packages/reploid/src/mesh/partitions/partition-stage-assignment.js +109 -0
- package/packages/reploid/src/mesh/partitions/partition-stage-commit.js +103 -0
- package/packages/reploid/src/mesh/partitions/partition-step-receiver.js +291 -0
- package/packages/reploid/src/mesh/partitions/partition-summary.js +122 -0
- package/packages/reploid/src/mesh/partitions/preparation-owner.js +40 -0
- package/packages/reploid/src/mesh/partitions/resident-error-codes.json +10 -0
- package/packages/reploid/src/mesh/partitions/resident-partition.js +333 -0
- package/packages/reploid/src/mesh/partitions/resident-route.js +46 -0
- package/packages/reploid/src/mesh/partitions/signed-resident-factory.js +59 -0
- package/packages/reploid/src/mesh/peer-generation.js +896 -0
- package/packages/reploid/src/mesh/peer-identity.js +87 -0
- package/packages/reploid/src/mesh/placement-beliefs.js +107 -0
- package/packages/reploid/src/mesh/preparation-progress.js +46 -0
- package/packages/reploid/src/mesh/provider-hardware.js +11 -0
- package/packages/reploid/src/mesh/remote-generation-requests.js +53 -0
- package/packages/reploid/src/mesh/swarm-coordination.js +140 -0
- package/packages/reploid/src/mesh/web-access-network.js +112 -0
- package/packages/reploid/src/mesh/web-access-policy.json +17 -0
- package/packages/reploid/src/retrieval/citations.js +156 -0
- package/packages/reploid/src/retrieval/claim-support.js +113 -0
- package/packages/reploid/src/retrieval/costs.js +101 -0
- package/packages/reploid/src/retrieval/fixed-provider.js +34 -0
- package/packages/reploid/src/retrieval/index.js +20 -0
- package/packages/reploid/src/retrieval/lexical-support.js +27 -0
- package/packages/reploid/src/retrieval/model-reviewer.js +67 -0
- package/packages/reploid/src/retrieval/page-fetch.json +10 -0
- package/packages/reploid/src/retrieval/page-text.js +182 -0
- package/packages/reploid/src/retrieval/passage.js +37 -0
- package/packages/reploid/src/retrieval/policy.json +14 -0
- package/packages/reploid/src/retrieval/retrieval.js +23 -0
- package/packages/reploid/src/retrieval/review-queue.js +76 -0
- package/packages/reploid/src/retrieval/sources.js +263 -0
- package/packages/reploid/src/retrieval/support-review.js +155 -0
- package/packages/reploid/src/retrieval/text.js +21 -0
- package/packages/reploid/src/rules/contribution.rules.json +8 -0
- package/packages/reploid/src/rules/routing.rules.json +24 -0
- package/packages/reploid/src/transport/assignment.js +571 -0
- package/packages/reploid/src/transport/bounded-channel-writer.js +46 -0
- package/packages/reploid/src/transport/index.js +11 -0
- package/packages/reploid/src/transport/introduction-admission.js +24 -0
- package/packages/reploid/src/transport/introduction-bootstrap.js +112 -0
- package/packages/reploid/src/transport/introduction-policy.json +12 -0
- package/packages/reploid/src/transport/introduction-records.js +90 -0
- package/packages/reploid/src/transport/introductions.js +209 -0
- package/packages/reploid/src/transport/partition-data-channel.js +421 -0
- package/packages/reploid/src/transport/partition-errors.json +16 -0
- package/packages/reploid/src/transport/retry-policy.js +36 -0
- package/packages/reploid/src/transport/room.js +502 -0
- package/packages/reploid/src/transport/sdp-fingerprint.js +5 -0
- package/packages/reploid/src/transport/signaling.js +571 -0
- package/packages/reploid/src/transport/swarm.js +1247 -0
- package/packages/reploid/src/transport/tool-offer-channel.js +137 -0
- package/server/browser-roots.js +42 -0
- package/server/browser-web-worker.js +94 -0
- package/server/chat-completions.js +45 -0
- package/server/fleet/browser-bridge.js +145 -0
- package/server/fleet/contract.js +42 -0
- package/server/fleet/credit-evidence.js +103 -0
- package/server/fleet/credit-recovery.js +175 -0
- package/server/fleet/credits.js +178 -0
- package/server/fleet/execution-admission.js +32 -0
- package/server/fleet/native-host.js +817 -0
- package/server/fleet/placement.js +132 -0
- package/server/fleet/requester.js +346 -0
- package/server/fleet/service.js +311 -0
- package/server/fleet/terminal-policy.js +83 -0
- package/server/fleet/text-api.js +94 -0
- package/server/hashing.js +35 -0
- package/server/http-request-owner.js +49 -0
- package/server/messages.js +57 -0
- package/server/native-canonical-epochs.js +90 -0
- package/server/native-change-log.js +40 -0
- package/server/native-earning.js +203 -0
- package/server/native-generation-journal.js +46 -0
- package/server/native-model-config.js +57 -0
- package/server/native-model-files.js +382 -0
- package/server/native-model-origin.js +54 -0
- package/server/native-model-origins.json +12 -0
- package/server/native-node.js +83 -0
- package/server/native-owner/controls.js +162 -0
- package/server/native-owner/monitor.js +251 -0
- package/server/native-owner/policy.js +273 -0
- package/server/native-owner/rate.js +92 -0
- package/server/native-participant.js +866 -0
- package/server/native-readiness-log.js +30 -0
- package/server/native-release-storage.js +64 -0
- package/server/native-shader-cache.js +369 -0
- package/server/native-supply-budget.js +66 -0
- package/server/native-whole-earning.js +67 -0
- package/server/native-whole-model-fill.js +182 -0
- package/server/native-whole-model.js +130 -0
- package/server/public-web-worker.js +66 -0
- package/server/responses.js +92 -0
- package/server/search/errors.js +47 -0
- package/server/search/page-fetch.js +185 -0
- package/server/text-api.js +83 -0
- package/server/text-recovery.js +45 -0
- package/server/text-request-options.js +23 -0
- package/server/text-requester.js +231 -0
- package/server/token-auth.js +9 -0
- package/server/whole-model-requester.js +99 -0
- package/shared/config/chat-models.json +4685 -0
- package/shared/config/doppler-execution-contracts.js +32 -0
- package/shared/config/doppler-execution-contracts.json +43 -0
- package/shared/config/doppler-generation-contract.js +223 -0
- package/shared/config/doppler-local-models.js +79 -0
- package/shared/config/doppler-package.json +7 -0
- package/shared/config/model-manifests/a0dc69e917133a82780823d17fc771c0f6b882d7c86c571deeaef45be2ff108b.json +6434 -0
- package/shared/config/model-manifests/e985e3cdad60d0cb57d84b436161e66ab5b3f6589a168de6f00b4610e5f1fb59.json +16534 -0
- package/shared/config/model-manifests/fa6eb7bbc3ba63571f8cede780b877d9272bde948697af93dd3d33837cb46137.json +7175 -0
- package/shared/config/model-pieces/qwen-0-8b-pieces.json +1506 -0
- package/shared/config/model-pieces/qwen-2b-pieces.json +4800 -0
- package/shared/config/model-pieces/qwen35b-q4_0-pieces.json +28880 -0
- package/shared/config/reploid-library.json +49 -0
- package/shared/config/swarm-bootstrap.json +21 -0
- package/shared/infrastructure/doppler-runtime-service.js +261 -0
- package/shared/pool/executable-pack.js +87 -0
- package/shared/pool/pack-operation-adapters.js +124 -0
- package/shared/pool/pack-operation-policy.js +75 -0
- package/shared/pool/pool-config.json +2280 -0
- package/shared/vendor/reploid/adapters/doppler-session.js +372 -0
- package/shared/vendor/reploid/artifacts/binary.js +25 -0
- package/shared/vendor/reploid/artifacts/canonical-json.js +23 -0
- package/shared/vendor/reploid/artifacts/delivery-attestation.js +64 -0
- package/shared/vendor/reploid/artifacts/identity.js +91 -0
- package/shared/vendor/reploid/artifacts/manifest-identity.js +17 -0
- package/shared/vendor/reploid/artifacts/participant-proofs.js +159 -0
- package/shared/vendor/reploid/artifacts/signatures.js +94 -0
- package/shared/vendor/reploid/config/defaults.json +110 -0
- package/shared/vendor/reploid/config/index.js +209 -0
- package/shared/vendor/reploid/config/request-overrides.json +9 -0
- package/shared/vendor/reploid/hashing.js +30 -0
- package/shared/vendor/reploid/mesh/generation-context-cache.js +97 -0
- package/shared/vendor/reploid/mesh/partitions/partition-attempt-evidence.js +282 -0
- package/shared/vendor/reploid/mesh/partitions/partition-contract.js +192 -0
- package/shared/vendor/reploid/mesh/partitions/partition-receiver-policy.json +7 -0
- package/shared/vendor/reploid/mesh/partitions/partition-route-ranking.js +268 -0
- package/shared/vendor/reploid/mesh/preparation-progress.js +46 -0
- package/shared/vendor/reploid/mesh/provider-hardware.js +11 -0
- package/shared/vendor/reploid/rules/routing.rules.json +24 -0
- package/shared/vendor/reploid/transport/partition-errors.json +16 -0
- package/README.md +0 -3
package/package.json
CHANGED
|
@@ -1,6 +1,31 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "reploid",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"
|
|
5
|
-
"description": "
|
|
6
|
-
|
|
3
|
+
"version": "0.1.0",
|
|
4
|
+
"type": "module",
|
|
5
|
+
"description": "Native delivery of the Reploid application",
|
|
6
|
+
"license": "MIT",
|
|
7
|
+
"engines": {
|
|
8
|
+
"node": ">=22.13.0"
|
|
9
|
+
},
|
|
10
|
+
"bin": {
|
|
11
|
+
"reploid": "./server/native-node.js",
|
|
12
|
+
"reploid-participant": "./server/native-participant.js",
|
|
13
|
+
"reploid-fleet-host": "./server/fleet/native-host.js"
|
|
14
|
+
},
|
|
15
|
+
"repository": {
|
|
16
|
+
"type": "git",
|
|
17
|
+
"url": "https://github.com/clocksmith/reploid.git"
|
|
18
|
+
},
|
|
19
|
+
"homepage": "https://github.com/clocksmith/reploid",
|
|
20
|
+
"scripts": {
|
|
21
|
+
"start": "node server/native-participant.js"
|
|
22
|
+
},
|
|
23
|
+
"dependencies": {
|
|
24
|
+
"doppler-gpu": "0.7.1",
|
|
25
|
+
"ws": "^8.21.0",
|
|
26
|
+
"@roamhq/wrtc": "0.10.0"
|
|
27
|
+
},
|
|
28
|
+
"optionalDependencies": {
|
|
29
|
+
"webgpu": "0.4.0"
|
|
30
|
+
}
|
|
31
|
+
}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
/** Binary encodings shared by signed artifacts and host transport adapters. */
|
|
2
|
+
export function bytesToBase64(value) {
|
|
3
|
+
const bytes = value instanceof Uint8Array ? value : new Uint8Array(value);
|
|
4
|
+
if (typeof Buffer !== 'undefined') return Buffer.from(bytes).toString('base64');
|
|
5
|
+
let binary = '';
|
|
6
|
+
for (let offset = 0; offset < bytes.length; offset += 0x8000) {
|
|
7
|
+
binary += String.fromCharCode(...bytes.subarray(offset, offset + 0x8000));
|
|
8
|
+
}
|
|
9
|
+
return btoa(binary);
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
export function base64ToBytes(value) {
|
|
13
|
+
const text = String(value || '');
|
|
14
|
+
if (typeof Buffer !== 'undefined') return new Uint8Array(Buffer.from(text, 'base64'));
|
|
15
|
+
return Uint8Array.from(atob(text), character => character.charCodeAt(0));
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export function toBase64Url(value) {
|
|
19
|
+
return bytesToBase64(value).replace(/\+/g, '-').replace(/\//g, '_').replace(/=+$/g, '');
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export function fromBase64Url(value) {
|
|
23
|
+
const text = String(value || '').replace(/-/g, '+').replace(/_/g, '/');
|
|
24
|
+
return base64ToBytes(text.padEnd(Math.ceil(text.length / 4) * 4, '='));
|
|
25
|
+
}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview Canonical JSON serialization shared across Poolday runtimes.
|
|
3
|
+
*/
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Canonical JSON whose object keys follow `compare`; without one, keys sort by
|
|
7
|
+
* UTF-16 code units, as Array.prototype.sort does.
|
|
8
|
+
*/
|
|
9
|
+
export function canonicalizeWith(compare) {
|
|
10
|
+
const serialize = (value) => {
|
|
11
|
+
if (value === null || typeof value !== 'object') return JSON.stringify(value);
|
|
12
|
+
if (Array.isArray(value)) {
|
|
13
|
+
return `[${value.map((item) => serialize(item)).join(',')}]`;
|
|
14
|
+
}
|
|
15
|
+
const keys = Object.keys(value).sort(compare);
|
|
16
|
+
return `{${keys.map((key) => `${JSON.stringify(key)}:${serialize(value[key])}`).join(',')}}`;
|
|
17
|
+
};
|
|
18
|
+
return serialize;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export const canonicalize = canonicalizeWith();
|
|
22
|
+
|
|
23
|
+
export default canonicalize;
|
|
@@ -0,0 +1,402 @@
|
|
|
1
|
+
import { signDeliveryAttestation } from '../delivery-attestation.js';
|
|
2
|
+
import { untilAborted } from '../../cancellation.js';
|
|
3
|
+
|
|
4
|
+
/** Discovery composition for the existing signed custody store and bounded byte channel. */
|
|
5
|
+
export function createCustodyExchange({ transport, identity, policy, ports }) {
|
|
6
|
+
policy = structuredClone(policy);
|
|
7
|
+
for (const name of ['maxTransfers', 'maxPeers', 'maxAcquisitionAttempts', 'maxInventoryFiles', 'maxSupplyBytes', 'maxArtifactBytes', 'grantMs']) {
|
|
8
|
+
if (!Number.isSafeInteger(policy[name]) || policy[name] <= 0) throw new Error('Invalid custody policy: ' + name);
|
|
9
|
+
}
|
|
10
|
+
const { createSupplier, createStore, createChannel, readArtifact, verifyArtifact, hash, hashBytes, checkpoints } = ports;
|
|
11
|
+
const now = ports.now || Date.now;
|
|
12
|
+
const budgetState = (state = ports.supplyBudget?.getState()) => {
|
|
13
|
+
if (!state || state.maxBytes !== policy.maxSupplyBytes || !Number.isSafeInteger(state.reservedBytes)
|
|
14
|
+
|| state.reservedBytes < 0 || state.reservedBytes > state.maxBytes) throw new Error('Persistent file contribution budget unavailable');
|
|
15
|
+
return state;
|
|
16
|
+
};
|
|
17
|
+
const peers = new Map(), pending = new Map(), suppliers = new Map(), channels = new Map(), channelWaiters = new Map();
|
|
18
|
+
const offerWaiters = new Set();
|
|
19
|
+
// A supplier that delivered nothing (silent past a deadline or disconnected) may answer a later grant.
|
|
20
|
+
const unavailable = message => Object.assign(new Error(message), { code: 'CUSTODY_SUPPLIER_UNAVAILABLE' });
|
|
21
|
+
const lifetime = new AbortController();
|
|
22
|
+
const operations = new Set(), supplyOperations = new Set();
|
|
23
|
+
const ownSupply = operation => {
|
|
24
|
+
operations.add(operation); supplyOperations.add(operation);
|
|
25
|
+
operation.finally(() => { operations.delete(operation); supplyOperations.delete(operation); }).catch(() => {});
|
|
26
|
+
return operation;
|
|
27
|
+
};
|
|
28
|
+
let offered = [], supply = false, supplyEpoch = 0, reserved = 0, preparing = 0, acquiring = 0, closed = false;
|
|
29
|
+
const key = artifact => JSON.stringify([artifact.path, artifact.hash, artifact.hashAlgorithm, artifact.sizeBytes]);
|
|
30
|
+
// Relative paths of plain segments: signed catalogs name program files under artifacts/... (full-model
|
|
31
|
+
// seeding offers them); never absolute, empty, '.' or '..' segments. Storage is keyed by hash, not path.
|
|
32
|
+
const safePath = path => typeof path === 'string' && path.length > 0 && path.length <= 256
|
|
33
|
+
&& path.split('/').every(segment => /^[\w.-]+$/.test(segment) && segment !== '.' && segment !== '..');
|
|
34
|
+
const valid = artifact => artifact && typeof artifact === 'object' && safePath(artifact.path)
|
|
35
|
+
&& typeof artifact.role === 'string' && artifact.role.length <= 64
|
|
36
|
+
&& typeof artifact.hash === 'string' && /^[a-f0-9]{64}$/.test(artifact.hash)
|
|
37
|
+
&& ['sha256', 'blake3'].includes(artifact.hashAlgorithm)
|
|
38
|
+
&& Number.isSafeInteger(artifact.sizeBytes) && artifact.sizeBytes > 0 && artifact.sizeBytes <= policy.maxArtifactBytes;
|
|
39
|
+
const live = () => { lifetime.signal.throwIfAborted(); };
|
|
40
|
+
const connected = () => {
|
|
41
|
+
const ids = new Set(transport.getConnectedPeers().map(peer => peer.id));
|
|
42
|
+
for (const peer of peers.keys()) if (!ids.has(peer)) peers.delete(peer);
|
|
43
|
+
return ids;
|
|
44
|
+
};
|
|
45
|
+
const notify = () => ports.onChange?.();
|
|
46
|
+
// Identity certificates travel beside requests and grants, never inside them, so grant hashes
|
|
47
|
+
// and older peers are unchanged. Only one bound to the party's grant key and peer id is kept;
|
|
48
|
+
// verifying who signed it belongs to whoever attributes the transfer.
|
|
49
|
+
const bound = (certificate, party) => certificate !== null && typeof certificate === 'object' && !Array.isArray(certificate)
|
|
50
|
+
&& certificate.custodyPeerId === party?.peerId && certificate.custodyPublicKey === party?.publicKey
|
|
51
|
+
&& JSON.stringify(certificate).length <= policy.channel.maxControlBytes;
|
|
52
|
+
const ownCertificate = async () => {
|
|
53
|
+
if (!ports.certificate) return null;
|
|
54
|
+
try {
|
|
55
|
+
const certificate = await ports.certificate();
|
|
56
|
+
return bound(certificate, identity) ? structuredClone(certificate) : null;
|
|
57
|
+
} catch (error) { ports.onError?.(error); return null; }
|
|
58
|
+
};
|
|
59
|
+
// A certified requester signs the exact completed receipt with the custody key its grant names,
|
|
60
|
+
// for whoever claims the delivery. An uncertified host's key names no one, so it signs nothing.
|
|
61
|
+
const attest = async (receipt, certificate) => {
|
|
62
|
+
if (!certificate) return null;
|
|
63
|
+
try { return await signDeliveryAttestation({ receipt, attestedAt: new Date(now()).toISOString(), privateKey: identity.privateKey, hash }); }
|
|
64
|
+
catch (error) { ports.onError?.(error); return null; }
|
|
65
|
+
};
|
|
66
|
+
// Optional host admission: it withholds new supply and its advertisement; granted transfers continue.
|
|
67
|
+
const admitted = (active, advertise) => !ports.admitSupply || ports.admitSupply({ active, advertise }) === true;
|
|
68
|
+
const announce = peer => {
|
|
69
|
+
const payload = { artifacts: supply && admitted(0, true) ? offered : [] };
|
|
70
|
+
if (peer) transport.sendToPeer(peer, 'reploid:custody-offer', payload);
|
|
71
|
+
else transport.broadcast('reploid:custody-offer', payload);
|
|
72
|
+
};
|
|
73
|
+
const install = (peer, channel) => {
|
|
74
|
+
if (closed || channels.has(peer) || channels.size >= policy.maxPeers || !connected().has(peer)) { channel.close(); return; }
|
|
75
|
+
let bus;
|
|
76
|
+
const ready = new Promise((resolve, reject) => {
|
|
77
|
+
const timeout = setTimeout(() => { channel.close(); reject(unavailable('Custody channel open timed out')); }, policy.channel.timeoutMs);
|
|
78
|
+
const open = () => {
|
|
79
|
+
clearTimeout(timeout);
|
|
80
|
+
try {
|
|
81
|
+
bus = createChannel({ channel, limits: policy.channel, serve: request => {
|
|
82
|
+
const entry = suppliers.get(request.transferId);
|
|
83
|
+
if (!supply || entry?.peer !== peer) throw new Error('File supply is not authorized');
|
|
84
|
+
return ownSupply(Promise.resolve(entry.supplier.serve(request)));
|
|
85
|
+
} });
|
|
86
|
+
resolve(bus);
|
|
87
|
+
} catch (error) { channel.close(); reject(error); }
|
|
88
|
+
};
|
|
89
|
+
if (channel.readyState === 'open') open(); else channel.addEventListener('open', open, { once: true });
|
|
90
|
+
channel.addEventListener('close', () => { clearTimeout(timeout); reject(unavailable('Custody peer disconnected')); }, { once: true });
|
|
91
|
+
});
|
|
92
|
+
const entry = { ready, close: () => { bus?.close(); channel.close(); } };
|
|
93
|
+
channels.set(peer, entry); ready.catch(() => {});
|
|
94
|
+
channelWaiters.get(peer)?.finish(null, ready);
|
|
95
|
+
channel.addEventListener('close', () => { if (channels.get(peer) === entry) channels.delete(peer); }, { once: true });
|
|
96
|
+
return entry;
|
|
97
|
+
};
|
|
98
|
+
const channelFor = (peer, signal) => {
|
|
99
|
+
signal.throwIfAborted();
|
|
100
|
+
let ready = channels.get(peer)?.ready;
|
|
101
|
+
// Exactly one endpoint opens the auxiliary channel, including simultaneous
|
|
102
|
+
// acquisitions in opposite directions. No channel glare or replacement.
|
|
103
|
+
if (!ready && identity.peerId < peer) ready = install(peer, transport.openDataChannel(peer, 'reploid-custody')).ready;
|
|
104
|
+
if (!ready) {
|
|
105
|
+
ready = channelWaiters.get(peer)?.ready;
|
|
106
|
+
if (!ready) {
|
|
107
|
+
ready = new Promise((resolve, reject) => {
|
|
108
|
+
const timer = setTimeout(() => finish(unavailable('Custody channel open timed out')), policy.channel.timeoutMs);
|
|
109
|
+
const finish = (error, result) => {
|
|
110
|
+
clearTimeout(timer); channelWaiters.delete(peer);
|
|
111
|
+
if (error) reject(error); else resolve(result);
|
|
112
|
+
};
|
|
113
|
+
channelWaiters.set(peer, { finish });
|
|
114
|
+
});
|
|
115
|
+
channelWaiters.get(peer).ready = ready;
|
|
116
|
+
if (!transport.sendToPeer(peer, 'reploid:custody-request', { channel: true })) {
|
|
117
|
+
channelWaiters.get(peer).finish(unavailable('File peer disconnected'));
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
return untilAborted(ready, signal);
|
|
122
|
+
};
|
|
123
|
+
const revokeSupply = () => {
|
|
124
|
+
supply = false; supplyEpoch++;
|
|
125
|
+
for (const entry of suppliers.values()) { clearTimeout(entry.timer); entry.supplier.close(); }
|
|
126
|
+
suppliers.clear();
|
|
127
|
+
};
|
|
128
|
+
const stopSupply = () => { revokeSupply(); announce(); notify(); return Promise.allSettled([...supplyOperations]); };
|
|
129
|
+
const unsub = transport.onDataChannel('reploid-custody', install);
|
|
130
|
+
transport.onMessage('reploid:custody-offer', (peer, message) => {
|
|
131
|
+
if (closed || !Array.isArray(message?.artifacts) || message.artifacts.length > policy.maxInventoryFiles
|
|
132
|
+
|| !message.artifacts.every(valid)) return;
|
|
133
|
+
if (!connected().has(peer) || !peers.has(peer) && peers.size >= policy.maxPeers) return;
|
|
134
|
+
const first = !peers.has(peer);
|
|
135
|
+
peers.set(peer, structuredClone(message.artifacts)); if (first) announce(peer); notify();
|
|
136
|
+
for (const wake of offerWaiters) wake();
|
|
137
|
+
});
|
|
138
|
+
transport.onMessage('reploid:custody-response', (peer, message) => {
|
|
139
|
+
const entry = pending.get(message?.id);
|
|
140
|
+
if (!entry || entry.peer !== peer) return;
|
|
141
|
+
entry.finish(message.error ? new Error(message.error) : null, message);
|
|
142
|
+
});
|
|
143
|
+
transport.onMessage('reploid:custody-request', (peer, message) => {
|
|
144
|
+
if (closed) return;
|
|
145
|
+
if (message?.channel === true) {
|
|
146
|
+
if (identity.peerId < peer && !channels.has(peer)
|
|
147
|
+
&& transport.getConnectedPeers().some(item => item.id === peer)) {
|
|
148
|
+
try { install(peer, transport.openDataChannel(peer, 'reploid-custody')); }
|
|
149
|
+
catch (error) { ports.onError?.(error); }
|
|
150
|
+
}
|
|
151
|
+
return;
|
|
152
|
+
}
|
|
153
|
+
if (message?.release) {
|
|
154
|
+
const entry = suppliers.get(message.release);
|
|
155
|
+
if (entry?.peer === peer) { entry.supplier.close(); clearTimeout(entry.timer); suppliers.delete(message.release); }
|
|
156
|
+
return;
|
|
157
|
+
}
|
|
158
|
+
const operation = (async () => {
|
|
159
|
+
live();
|
|
160
|
+
if (typeof message?.id !== 'string' || message.id.length > 128 || !valid(message.artifact)) throw new Error('Invalid file request');
|
|
161
|
+
if (message.maxExpiresAt !== undefined && !Number.isSafeInteger(message.maxExpiresAt)) throw new Error('Invalid file expiry limit');
|
|
162
|
+
if (!supply || preparing + suppliers.size >= policy.maxTransfers
|
|
163
|
+
|| !admitted(preparing + suppliers.size, false)) throw new Error('File contribution unavailable');
|
|
164
|
+
const descriptor = offered.find(item => key(item) === key(message.artifact));
|
|
165
|
+
if (!descriptor || message.requester?.peerId !== peer || !message.requester.publicKey) throw new Error('Unapproved artifact request');
|
|
166
|
+
const selected = rangeDescriptor(descriptor, message.range);
|
|
167
|
+
if (!Number.isSafeInteger(descriptor.sizeBytes) || descriptor.sizeBytes > policy.maxArtifactBytes
|
|
168
|
+
|| reserved + selected.sizeBytes > policy.maxSupplyBytes) throw new Error('File contribution limit reached');
|
|
169
|
+
const epoch = supplyEpoch; preparing++;
|
|
170
|
+
try {
|
|
171
|
+
budgetState();
|
|
172
|
+
const reservation = await ports.supplyBudget.reserve(selected.sizeBytes);
|
|
173
|
+
reserved = Math.max(reserved, budgetState(reservation).reservedBytes);
|
|
174
|
+
budgetState(); live();
|
|
175
|
+
if (!supply || epoch !== supplyEpoch) throw new Error('File contribution stopped');
|
|
176
|
+
let bytes = await readArtifact(descriptor, { signal: lifetime.signal });
|
|
177
|
+
await verifyArtifact(descriptor, bytes); live();
|
|
178
|
+
if (message.range) {
|
|
179
|
+
bytes = bytes.slice(message.range.offset, message.range.offset + message.range.size);
|
|
180
|
+
await verifyArtifact(selected, bytes);
|
|
181
|
+
}
|
|
182
|
+
if (!supply || epoch !== supplyEpoch) throw new Error('File contribution stopped');
|
|
183
|
+
const artifact = { artifactId: selected.path, path: selected.path, role: selected.role,
|
|
184
|
+
sizeBytes: bytes.byteLength, hash: await hashBytes(bytes) };
|
|
185
|
+
const artifacts = [artifact], chunks = [];
|
|
186
|
+
for (let offset = 0; offset < bytes.byteLength; offset += policy.channel.maxChunkBytes) {
|
|
187
|
+
const chunk = bytes.subarray(offset, offset + policy.channel.maxChunkBytes);
|
|
188
|
+
chunks.push({ index: chunks.length, offset, sizeBytes: chunk.byteLength, hash: await hashBytes(chunk) });
|
|
189
|
+
}
|
|
190
|
+
const artifactSet = { schema: 'reploid.pool.artifact-set/v1', identity: await hash(artifacts), artifacts };
|
|
191
|
+
const index = { schema: 'reploid.pool.pack-custody-index/v2', artifactSetIdentity: artifactSet.identity,
|
|
192
|
+
artifacts: [{ artifactId: artifact.artifactId, hash: artifact.hash, sizeBytes: artifact.sizeBytes, chunks }] };
|
|
193
|
+
// Both participants bound lifetime. Echoing the requester's absolute
|
|
194
|
+
// ceiling avoids comparing two independently minted full-duration grants.
|
|
195
|
+
const supplierExpiresAt = now() + policy.grantMs;
|
|
196
|
+
const authorization = { schema: 'reploid.pool.pack-custody-authorization/v2', artifactSet,
|
|
197
|
+
transferId: crypto.randomUUID(), attempt: 1,
|
|
198
|
+
expiresAt: Math.min(supplierExpiresAt, message.maxExpiresAt ?? supplierExpiresAt),
|
|
199
|
+
requester: structuredClone(message.requester), suppliers: [{ peerId: identity.peerId, publicKey: identity.publicKey }],
|
|
200
|
+
indexDigest: await hash(index), limits: { maxArtifactBytes: policy.maxArtifactBytes,
|
|
201
|
+
maxChunkBytes: policy.channel.maxChunkBytes, maxTransferBytes: selected.sizeBytes,
|
|
202
|
+
maxConcurrentChunks: policy.channel.maxPendingRequests,
|
|
203
|
+
requestTimeoutMs: policy.channel.timeoutMs } };
|
|
204
|
+
const supplier = await createSupplier({ authorization, index, peerId: identity.peerId, privateKey: identity.privateKey,
|
|
205
|
+
inventory: { expiresAt: authorization.expiresAt, maxBytes: selected.sizeBytes,
|
|
206
|
+
artifacts: [{ artifactId: artifact.artifactId, chunkIndexes: chunks.map(chunk => chunk.index) }] },
|
|
207
|
+
readChunk: async (_, chunk) => bytes.slice(chunk.offset, chunk.offset + chunk.sizeBytes), now });
|
|
208
|
+
const certificates = { requester: bound(message.requesterCertificate, message.requester) ? structuredClone(message.requesterCertificate) : null,
|
|
209
|
+
supplier: await ownCertificate() };
|
|
210
|
+
if (closed || !supply || epoch !== supplyEpoch) { supplier.close(); throw new Error('File contribution stopped'); }
|
|
211
|
+
const timer = setTimeout(() => { supplier.close(); suppliers.delete(authorization.transferId); }, Math.max(0, authorization.expiresAt - now()));
|
|
212
|
+
suppliers.set(authorization.transferId, { supplier, peer, timer });
|
|
213
|
+
try {
|
|
214
|
+
if (!transport.sendToPeer(peer, 'reploid:custody-response', { id: message.id, authorization, index, inventory: supplier.inventory,
|
|
215
|
+
...(certificates.supplier ? { supplierCertificate: certificates.supplier } : {}) })) {
|
|
216
|
+
throw new Error('File offer response could not be queued');
|
|
217
|
+
}
|
|
218
|
+
} catch (error) {
|
|
219
|
+
// A rejected send owns no live grant. Keep the durable reservation
|
|
220
|
+
// charged, but release supplier bytes and the simultaneous-transfer slot.
|
|
221
|
+
clearTimeout(timer); suppliers.delete(authorization.transferId); supplier.close();
|
|
222
|
+
throw error;
|
|
223
|
+
}
|
|
224
|
+
ports.observeDelivery?.({ role: 'supplier', transferId: authorization.transferId, peerId: peer,
|
|
225
|
+
authorization: structuredClone(authorization), index: structuredClone(index), receipt: null, certificates, attestation: null });
|
|
226
|
+
} finally { preparing--; notify(); }
|
|
227
|
+
})().catch(error => {
|
|
228
|
+
if (!closed) transport.sendToPeer(peer, 'reploid:custody-response', { id: message?.id, error: error.message });
|
|
229
|
+
});
|
|
230
|
+
ownSupply(operation);
|
|
231
|
+
});
|
|
232
|
+
const rangeDescriptor = (artifact, range) => {
|
|
233
|
+
if (range == null) return artifact;
|
|
234
|
+
if (!Number.isSafeInteger(range.offset) || range.offset < 0 || !Number.isSafeInteger(range.size)
|
|
235
|
+
|| range.size < 1 || range.offset + range.size > artifact.sizeBytes
|
|
236
|
+
|| !/^sha256:[a-f0-9]{64}$/.test(range.identity)) throw new Error('Invalid verified file range');
|
|
237
|
+
return { path: `piece-${range.identity.slice(7)}.bin`, role: artifact.role,
|
|
238
|
+
sizeBytes: range.size, hash: range.identity.slice(7), hashAlgorithm: 'sha256' };
|
|
239
|
+
};
|
|
240
|
+
const request = (peer, artifact, signal, range, maxExpiresAt, requesterCertificate) => new Promise((resolve, reject) => {
|
|
241
|
+
signal.throwIfAborted();
|
|
242
|
+
if (pending.size >= policy.maxTransfers) { reject(new Error('File acquisition limit reached')); return; }
|
|
243
|
+
const id = crypto.randomUUID();
|
|
244
|
+
const finish = (error, result) => {
|
|
245
|
+
if (!pending.delete(id)) return;
|
|
246
|
+
clearTimeout(timer); signal.removeEventListener('abort', abort);
|
|
247
|
+
if (error) reject(error); else resolve(result);
|
|
248
|
+
};
|
|
249
|
+
const abort = () => finish(signal.reason);
|
|
250
|
+
const timer = setTimeout(() => finish(unavailable('File offer timed out')), policy.channel.timeoutMs);
|
|
251
|
+
pending.set(id, { peer, finish }); signal.addEventListener('abort', abort, { once: true });
|
|
252
|
+
if (!transport.sendToPeer(peer, 'reploid:custody-request', { id, artifact, range, maxExpiresAt,
|
|
253
|
+
requester: { peerId: identity.peerId, publicKey: identity.publicKey },
|
|
254
|
+
...(requesterCertificate ? { requesterCertificate } : {}) })) finish(unavailable('File peer disconnected'));
|
|
255
|
+
});
|
|
256
|
+
const acquireFrom = async (peer, source, combined, range, reserveDownload) => {
|
|
257
|
+
const artifact = rangeDescriptor(source, range);
|
|
258
|
+
const bus = await channelFor(peer, combined);
|
|
259
|
+
const maxExpiresAt = now() + policy.grantMs, requesterCertificate = await ownCertificate();
|
|
260
|
+
const result = await request(peer, source, combined, range, maxExpiresAt, requesterCertificate); combined.throwIfAborted();
|
|
261
|
+
const grant = result.authorization, declared = grant?.artifactSet?.artifacts?.[0];
|
|
262
|
+
if (grant?.artifactSet?.artifacts?.length !== 1 || declared?.path !== artifact.path || declared?.sizeBytes !== artifact.sizeBytes
|
|
263
|
+
|| grant.requester?.peerId !== identity.peerId || grant.requester?.publicKey !== identity.publicKey
|
|
264
|
+
|| grant.suppliers?.length !== 1 || grant.suppliers[0].peerId !== peer
|
|
265
|
+
|| grant.limits.maxArtifactBytes > policy.maxArtifactBytes || grant.limits.maxChunkBytes > policy.channel.maxChunkBytes
|
|
266
|
+
|| grant.limits.maxTransferBytes > artifact.sizeBytes || grant.limits.requestTimeoutMs > policy.channel.timeoutMs
|
|
267
|
+
|| grant.expiresAt > maxExpiresAt) throw new Error('File grant exceeds requested acquisition');
|
|
268
|
+
let store, reservationFailure;
|
|
269
|
+
try {
|
|
270
|
+
store = await createStore({ authorization: grant, index: result.index, inventories: [result.inventory],
|
|
271
|
+
requesterPrivateKey: identity.privateKey, requestChunk: async (_, message, controls) => {
|
|
272
|
+
// Checkpoint hits never call this port. Reserve every requested chunk,
|
|
273
|
+
// including failed attempts/retries, before any payload can arrive.
|
|
274
|
+
controls.signal.throwIfAborted();
|
|
275
|
+
if (reservationFailure) throw reservationFailure;
|
|
276
|
+
if (reserveDownload) {
|
|
277
|
+
try { await reserveDownload(message.sizeBytes); }
|
|
278
|
+
catch (cause) { reservationFailure = cause; throw cause; }
|
|
279
|
+
}
|
|
280
|
+
controls.signal.throwIfAborted();
|
|
281
|
+
return bus.requestChunk(message, controls);
|
|
282
|
+
},
|
|
283
|
+
checkpoints, signal: combined, now,
|
|
284
|
+
maxConcurrentChunks: Math.min(grant.limits.maxConcurrentChunks ?? 1, policy.channel.maxPendingRequests) });
|
|
285
|
+
const bytes = await store.readArtifact(declared);
|
|
286
|
+
await verifyArtifact(artifact, bytes); combined.throwIfAborted();
|
|
287
|
+
if (ports.commitArtifact) {
|
|
288
|
+
try { await ports.commitArtifact(artifact, bytes, { signal: combined }); }
|
|
289
|
+
catch (cause) {
|
|
290
|
+
const error = new Error('Verified file could not be retained: ' + cause.message, { cause });
|
|
291
|
+
error.code = 'CUSTODY_LOCAL_COMMIT_FAILED'; throw error;
|
|
292
|
+
}
|
|
293
|
+
// Complete verified files now own durability. Keep checkpoints only
|
|
294
|
+
// for interrupted transfers, not a second copy of every model file.
|
|
295
|
+
for (const chunk of result.index.artifacts[0].chunks) {
|
|
296
|
+
try { await checkpoints?.deleteChunk(chunk, { signal: combined }); }
|
|
297
|
+
catch (error) { ports.onError?.(error); }
|
|
298
|
+
}
|
|
299
|
+
combined.throwIfAborted();
|
|
300
|
+
}
|
|
301
|
+
ports.observe?.(store.getReceipt());
|
|
302
|
+
if (ports.observeDelivery) {
|
|
303
|
+
const receipt = structuredClone(store.getReceipt());
|
|
304
|
+
ports.observeDelivery({ role: 'requester', transferId: grant.transferId, peerId: peer,
|
|
305
|
+
authorization: structuredClone(grant), index: structuredClone(result.index), receipt,
|
|
306
|
+
certificates: { requester: requesterCertificate,
|
|
307
|
+
supplier: bound(result.supplierCertificate, grant.suppliers[0]) ? structuredClone(result.supplierCertificate) : null },
|
|
308
|
+
attestation: await attest(receipt, requesterCertificate) });
|
|
309
|
+
}
|
|
310
|
+
return bytes;
|
|
311
|
+
} catch (error) {
|
|
312
|
+
if (store) ports.observe?.({ ...store.getReceipt(), failure: error.message }); throw reservationFailure || error;
|
|
313
|
+
} finally {
|
|
314
|
+
store?.close(); transport.sendToPeer(peer, 'reploid:custody-request', { release: grant.transferId });
|
|
315
|
+
}
|
|
316
|
+
};
|
|
317
|
+
const candidates = artifact => {
|
|
318
|
+
const ids = connected();
|
|
319
|
+
return [...peers.entries()].filter(([peer, files]) => ids.has(peer) && files.some(item => key(item) === key(artifact)))
|
|
320
|
+
.map(([peer]) => peer).slice(0, policy.maxAcquisitionAttempts);
|
|
321
|
+
};
|
|
322
|
+
// Waits at most one channel timeout for an eligible supplier to offer the file again, for
|
|
323
|
+
// example after its connection is re-established; resolves to the eligible suppliers, possibly none.
|
|
324
|
+
const reoffered = (artifact, excluded, signal) => new Promise((resolve, reject) => {
|
|
325
|
+
const eligible = () => candidates(artifact).filter(peer => !excluded.has(peer));
|
|
326
|
+
const done = (error, value) => {
|
|
327
|
+
clearTimeout(timer); offerWaiters.delete(check); signal.removeEventListener('abort', abort);
|
|
328
|
+
if (error) reject(error); else resolve(value);
|
|
329
|
+
};
|
|
330
|
+
const check = () => { const ids = eligible(); if (ids.length) done(null, ids); };
|
|
331
|
+
const abort = () => done(signal.reason);
|
|
332
|
+
const timer = setTimeout(() => done(null, eligible()), policy.channel.timeoutMs);
|
|
333
|
+
offerWaiters.add(check); signal.addEventListener('abort', abort, { once: true });
|
|
334
|
+
if (signal.aborted) abort(); else check();
|
|
335
|
+
});
|
|
336
|
+
return Object.freeze({
|
|
337
|
+
announce,
|
|
338
|
+
getState: () => { connected(); return { sharing: supply, suppliedBytes: Math.max(reserved, ports.supplyBudget?.getState()?.reservedBytes || 0), pending: acquiring, peers: [...peers.keys()] }; },
|
|
339
|
+
has: artifact => candidates(artifact).length > 0,
|
|
340
|
+
offer(artifacts) {
|
|
341
|
+
if (!Array.isArray(artifacts) || artifacts.length > policy.maxInventoryFiles || !artifacts.every(valid)) throw new Error('Invalid file inventory');
|
|
342
|
+
live();
|
|
343
|
+
reserved = budgetState().reservedBytes;
|
|
344
|
+
const replacement = structuredClone(artifacts);
|
|
345
|
+
// Revoke old grants, but publish only the complete replacement. A transient
|
|
346
|
+
// empty inventory can make peers reject acquisition of a still-offered file.
|
|
347
|
+
// Inventory changes and stop/restart do not renew this owner's allowance.
|
|
348
|
+
revokeSupply(); offered = replacement; supply = true; announce(); notify();
|
|
349
|
+
},
|
|
350
|
+
stopSupply,
|
|
351
|
+
async acquire(artifact, { signal, range = null, reserveDownload = null }) {
|
|
352
|
+
live();
|
|
353
|
+
if (reserveDownload !== null && typeof reserveDownload !== 'function') throw new Error('Invalid acquisition accounting port');
|
|
354
|
+
if (!valid(artifact)) throw new Error('Invalid file acquisition');
|
|
355
|
+
range = range == null ? null : structuredClone(range);
|
|
356
|
+
rangeDescriptor(artifact, range);
|
|
357
|
+
if (acquiring >= policy.maxTransfers) throw new Error('File acquisition limit reached');
|
|
358
|
+
const combined = AbortSignal.any([lifetime.signal, signal]);
|
|
359
|
+
combined.throwIfAborted();
|
|
360
|
+
const descriptor = structuredClone(artifact), eligible = candidates(descriptor);
|
|
361
|
+
if (!eligible.length) throw new Error('No authorized peer offers this file');
|
|
362
|
+
acquiring++;
|
|
363
|
+
const operation = (async () => {
|
|
364
|
+
// At most maxAcquisitionAttempts grants, each fresh. A supplier that delivered nothing may be
|
|
365
|
+
// asked again (least-tried first); one that sent invalid data or refused is not. Verified chunks
|
|
366
|
+
// resume from checkpoints, and every requested chunk is charged, retries included.
|
|
367
|
+
const failures = [], tries = new Map(), excluded = new Set();
|
|
368
|
+
let suppliers = eligible;
|
|
369
|
+
while (suppliers.length) {
|
|
370
|
+
combined.throwIfAborted();
|
|
371
|
+
const peer = suppliers.reduce((best, id) => ((tries.get(id) || 0) < (tries.get(best) || 0) ? id : best));
|
|
372
|
+
tries.set(peer, (tries.get(peer) || 0) + 1);
|
|
373
|
+
try { return await acquireFrom(peer, descriptor, combined, range, reserveDownload); }
|
|
374
|
+
catch (error) {
|
|
375
|
+
combined.throwIfAborted();
|
|
376
|
+
if (error.code === 'CUSTODY_LOCAL_COMMIT_FAILED' || error.code?.startsWith('CUSTODY_BUDGET_')) throw error;
|
|
377
|
+
failures.push(error);
|
|
378
|
+
if (error.code !== 'CUSTODY_SUPPLIER_UNAVAILABLE') excluded.add(peer);
|
|
379
|
+
}
|
|
380
|
+
if (failures.length >= policy.maxAcquisitionAttempts) break;
|
|
381
|
+
suppliers = candidates(descriptor).filter(peer => !excluded.has(peer));
|
|
382
|
+
if (!suppliers.length && failures.at(-1).code === 'CUSTODY_SUPPLIER_UNAVAILABLE') suppliers = await reoffered(descriptor, excluded, combined);
|
|
383
|
+
}
|
|
384
|
+
if (failures.length === 1) throw failures[0];
|
|
385
|
+
throw new AggregateError(failures, 'Available file suppliers failed: ' + failures.map(error => error.message).join('; '));
|
|
386
|
+
})();
|
|
387
|
+
operations.add(operation); notify();
|
|
388
|
+
try { return await operation; }
|
|
389
|
+
finally { acquiring--; operations.delete(operation); notify(); }
|
|
390
|
+
},
|
|
391
|
+
close() {
|
|
392
|
+
if (closed) return Promise.allSettled([...operations]);
|
|
393
|
+
closed = true; stopSupply(); lifetime.abort(new Error('File exchange closed')); unsub();
|
|
394
|
+
for (const entry of pending.values()) entry.finish(lifetime.signal.reason);
|
|
395
|
+
for (const entry of channelWaiters.values()) entry.finish(lifetime.signal.reason);
|
|
396
|
+
offerWaiters.clear();
|
|
397
|
+
for (const channel of channels.values()) channel.close();
|
|
398
|
+
peers.clear(); channels.clear();
|
|
399
|
+
return Promise.allSettled([...operations]);
|
|
400
|
+
}
|
|
401
|
+
});
|
|
402
|
+
}
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
/** Coordinates verified acquisition across eligible sources. Existing custody
|
|
2
|
+
* exchanges can supply read(piece); storage and source credentials are host ports.
|
|
3
|
+
*/
|
|
4
|
+
import { untilAborted } from '../../cancellation.js';
|
|
5
|
+
|
|
6
|
+
export function createPieceAcquisition({ sources, cache, authorize, verify, limits, foregroundIdle }) {
|
|
7
|
+
if (typeof sources !== 'function' || typeof authorize !== 'function' || typeof verify !== 'function'
|
|
8
|
+
|| typeof foregroundIdle !== 'function' || !cache?.get || !cache?.put
|
|
9
|
+
|| !['maxConcurrent', 'maxQueued', 'maxPieceBytes', 'timeoutMs'].every(k => Number.isSafeInteger(limits?.[k]) && limits[k] > 0)) {
|
|
10
|
+
throw new Error('Explicit acquisition ports and limits required');
|
|
11
|
+
}
|
|
12
|
+
const policy = structuredClone(limits), tasks = new Map(), queue = [];
|
|
13
|
+
const lifetime = new AbortController();
|
|
14
|
+
const receipt = { transferredBytes: 0, reusedBytes: 0, completedPieces: 0, failedSources: [], active: 0 };
|
|
15
|
+
const pump = () => {
|
|
16
|
+
while (!lifetime.signal.aborted && queue.length && receipt.active < policy.maxConcurrent) {
|
|
17
|
+
const task = queue.shift(); receipt.active++;
|
|
18
|
+
run(task).then(task.resolve, task.reject).finally(() => {
|
|
19
|
+
receipt.active--; tasks.delete(task.piece.identity); pump();
|
|
20
|
+
});
|
|
21
|
+
}
|
|
22
|
+
};
|
|
23
|
+
async function run({ piece }) {
|
|
24
|
+
const signal = AbortSignal.any([lifetime.signal, AbortSignal.timeout(policy.timeoutMs)]);
|
|
25
|
+
const saved = await cache.get(piece.identity);
|
|
26
|
+
if (saved && await verify(piece, saved)) { receipt.reusedBytes += saved.byteLength; return saved; }
|
|
27
|
+
const failures = [];
|
|
28
|
+
for (const source of sources(piece)) {
|
|
29
|
+
signal.throwIfAborted();
|
|
30
|
+
if (await authorize({ sourceId: source.id, piece }) !== true) continue;
|
|
31
|
+
try {
|
|
32
|
+
await foregroundIdle(signal); signal.throwIfAborted();
|
|
33
|
+
const data = await source.read(piece, { signal });
|
|
34
|
+
receipt.transferredBytes += data.byteLength;
|
|
35
|
+
signal.throwIfAborted();
|
|
36
|
+
if (data.byteLength !== piece.size || !await verify(piece, data)) throw new Error('Piece verification failed');
|
|
37
|
+
await cache.put(piece.identity, data); receipt.completedPieces++;
|
|
38
|
+
return data;
|
|
39
|
+
} catch (error) {
|
|
40
|
+
signal.throwIfAborted(); failures.push(error);
|
|
41
|
+
receipt.failedSources.push({ sourceId: source.id, identity: piece.identity, error: String(error.message) });
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
throw new AggregateError(failures, 'No eligible source supplied the verified piece');
|
|
45
|
+
}
|
|
46
|
+
return Object.freeze({
|
|
47
|
+
acquire(piece, { signal } = {}) {
|
|
48
|
+
signal?.throwIfAborted(); lifetime.signal.throwIfAborted();
|
|
49
|
+
if (!/^sha256:[a-f0-9]{64}$/.test(piece?.identity) || !Number.isSafeInteger(piece.size)
|
|
50
|
+
|| piece.size < 1 || piece.size > policy.maxPieceBytes) throw new Error('Invalid piece allocation');
|
|
51
|
+
let task = tasks.get(piece.identity);
|
|
52
|
+
if (!task) {
|
|
53
|
+
if (tasks.size >= policy.maxQueued) throw new Error('Acquisition queue is full');
|
|
54
|
+
task = { piece: structuredClone(piece) };
|
|
55
|
+
task.promise = new Promise((resolve, reject) => { task.resolve = resolve; task.reject = reject; });
|
|
56
|
+
tasks.set(piece.identity, task); queue.push(task); pump();
|
|
57
|
+
} else if (task.piece.size !== piece.size) throw new Error('Conflicting piece identity');
|
|
58
|
+
// Cancelling one consumer does not discard a shared verified transfer.
|
|
59
|
+
const shared = task.promise;
|
|
60
|
+
if (!signal) return shared.then(bytes => bytes.slice());
|
|
61
|
+
return untilAborted(shared, signal).then(bytes => bytes.slice());
|
|
62
|
+
},
|
|
63
|
+
getReceipt: () => structuredClone(receipt),
|
|
64
|
+
async close() {
|
|
65
|
+
lifetime.abort(new Error('Acquisition closed'));
|
|
66
|
+
for (const task of queue.splice(0)) { task.reject(lifetime.signal.reason); tasks.delete(task.piece.identity); }
|
|
67
|
+
await Promise.allSettled([...tasks.values()].map(task => task.promise));
|
|
68
|
+
},
|
|
69
|
+
});
|
|
70
|
+
}
|