sandcms 0.6.3 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -11
- package/dist/admin/Account.svelte +2 -0
- package/dist/admin/Admin.svelte +21 -6
- package/dist/admin/AssetAttachment.svelte +9 -6
- package/dist/admin/AssetAttachment.svelte.d.ts +3 -1
- package/dist/admin/BlueprintFields.svelte +0 -3
- package/dist/admin/BlueprintFields.svelte.d.ts +0 -1
- package/dist/admin/Editor.svelte +5 -10
- package/dist/admin/Fields.svelte +2 -8
- package/dist/admin/Fields.svelte.d.ts +0 -1
- package/dist/admin/Gallery.svelte +221 -80
- package/dist/admin/Gallery.svelte.d.ts +0 -1
- package/dist/admin/Inbox.svelte +3 -2
- package/dist/admin/LinkField.svelte +0 -3
- package/dist/admin/LinkField.svelte.d.ts +0 -1
- package/dist/admin/MediaLibrary.svelte +10 -7
- package/dist/admin/MediaPicker.svelte +43 -35
- package/dist/admin/MediaPicker.svelte.d.ts +0 -1
- package/dist/admin/Navigation.svelte +13 -2
- package/dist/admin/Pages.svelte +195 -0
- package/dist/admin/Pages.svelte.d.ts +7 -0
- package/dist/admin/Shell.svelte +27 -5
- package/dist/admin/SortableList.svelte +82 -61
- package/dist/admin/SortableList.svelte.d.ts +3 -1
- package/dist/admin/ThemePicker.svelte +72 -0
- package/dist/admin/ThemePicker.svelte.d.ts +18 -0
- package/dist/admin/UploadButton.svelte +2 -1
- package/dist/admin/UploadProgress.svelte +24 -0
- package/dist/admin/UploadProgress.svelte.d.ts +7 -0
- package/dist/admin/Users.svelte +8 -3
- package/dist/admin/client.d.ts +3 -670
- package/dist/admin/client.js +2 -1
- package/dist/admin/components/ui/alert/alert.svelte.d.ts +3 -13
- package/dist/admin/components/ui/attachment/attachment-media.svelte.d.ts +3 -13
- package/dist/admin/components/ui/attachment/attachment.svelte.d.ts +6 -26
- package/dist/admin/components/ui/badge/badge.svelte.d.ts +7 -25
- package/dist/admin/components/ui/button/button.svelte.d.ts +15 -53
- package/dist/admin/components/ui/toggle/toggle.svelte.d.ts +6 -26
- package/dist/admin/generated/admin.css +823 -115
- package/dist/admin/navigation.d.ts +9 -0
- package/dist/admin/navigation.js +20 -1
- package/dist/admin/theme-context.d.ts +7 -0
- package/dist/admin/theme-context.js +3 -0
- package/dist/admin/theme.d.ts +7 -0
- package/dist/admin/theme.js +14 -0
- package/dist/admin/types.d.ts +3 -1
- package/dist/admin/uploads.d.ts +8 -1
- package/dist/admin/uploads.js +94 -16
- package/dist/blueprints/files.js +1 -0
- package/dist/blueprints/labels.d.ts +6 -0
- package/dist/blueprints/labels.js +14 -0
- package/dist/blueprints/load.js +16 -2
- package/dist/blueprints/types.d.ts +4 -1
- package/dist/cli/admin.d.ts +39 -0
- package/dist/cli/admin.js +95 -0
- package/dist/cli/backup.js +4 -4
- package/dist/cli/doctor.d.ts +10 -1
- package/dist/cli/doctor.js +98 -33
- package/dist/cli/email-test.d.ts +3 -0
- package/dist/cli/email-test.js +20 -0
- package/dist/cli/files.d.ts +2 -6
- package/dist/cli/files.js +12 -2
- package/dist/cli/index.js +401 -95
- package/dist/cli/rebaseline.d.ts +10 -0
- package/dist/cli/rebaseline.js +117 -0
- package/dist/cli/scaffold.js +8 -8
- package/dist/cli/setup.js +16 -26
- package/dist/cli/upgrade.js +34 -7
- package/dist/env.d.ts +26 -2
- package/dist/env.js +48 -6
- package/dist/notification-fields.js +2 -1
- package/dist/server/admin-load.d.ts +85 -5
- package/dist/server/admin-load.js +26 -1
- package/dist/server/api-access.d.ts +4 -0
- package/dist/server/api-access.js +15 -2
- package/dist/server/api-keys.d.ts +96 -0
- package/dist/server/api-keys.js +115 -0
- package/dist/server/api-pagination.d.ts +3 -0
- package/dist/server/api.js +21 -63
- package/dist/server/auth.d.ts +10 -9
- package/dist/server/cms.d.ts +63 -21
- package/dist/server/cms.js +64 -7
- package/dist/server/db/auto-migrate.d.ts +0 -4
- package/dist/server/db/auto-migrate.js +12 -3
- package/dist/server/db/legacy.d.ts +13 -0
- package/dist/server/db/legacy.js +60 -0
- package/dist/server/db/migration-data.js +2 -34
- package/dist/server/db/migrations/{0000_foundation.sql → 0000_baseline.sql} +120 -4
- package/dist/server/db/migrations/meta/0000_snapshot.json +578 -3
- package/dist/server/db/migrations/meta/_journal.json +2 -58
- package/dist/server/db/schema.d.ts +119 -2
- package/dist/server/db/schema.js +23 -9
- package/dist/server/db/store.d.ts +25 -2
- package/dist/server/db/store.js +90 -26
- package/dist/server/forms.js +2 -1
- package/dist/server/hooks.js +155 -81
- package/dist/server/media/contracts.d.ts +115 -0
- package/dist/server/media/contracts.js +12 -0
- package/dist/server/media/deletions.d.ts +2 -1
- package/dist/server/media/deletions.js +3 -3
- package/dist/server/media/delivery.js +7 -11
- package/dist/server/media/processor-child.d.ts +2 -0
- package/dist/server/media/processor-child.js +236 -0
- package/dist/server/media/processor.d.ts +51 -0
- package/dist/server/media/processor.js +135 -0
- package/dist/server/media/render-math.d.ts +24 -0
- package/dist/server/media/render-math.js +50 -0
- package/dist/server/media/runner.d.ts +30 -1
- package/dist/server/media/runner.js +418 -165
- package/dist/server/media/service.d.ts +53 -11
- package/dist/server/media/service.js +307 -98
- package/dist/server/media/storage.d.ts +5 -0
- package/dist/server/media/storage.js +279 -0
- package/dist/server/media/upload-ticket.d.ts +12 -0
- package/dist/server/media/upload-ticket.js +35 -0
- package/dist/server/media/verify.d.ts +64 -0
- package/dist/server/media/verify.js +201 -0
- package/dist/server/platform/http.d.ts +27 -0
- package/dist/server/platform/http.js +77 -0
- package/dist/server/platform/index.d.ts +52 -0
- package/dist/server/platform/index.js +98 -0
- package/dist/server/platform/snapshots.d.ts +18 -0
- package/dist/server/platform/snapshots.js +110 -0
- package/dist/server/platform/volume.d.ts +20 -0
- package/dist/server/platform/volume.js +60 -0
- package/docs/acceptance.md +7 -5
- package/docs/api.md +56 -4
- package/docs/cloudflare.md +26 -152
- package/docs/content.md +20 -5
- package/docs/deployment.md +222 -57
- package/docs/email.md +6 -0
- package/docs/forms.md +2 -2
- package/docs/media.md +94 -21
- package/docs/package.md +21 -18
- package/docs/plans/0.7-self-hosted.md +264 -0
- package/docs/releases.md +37 -1
- package/docs/upgrades.md +72 -9
- package/package.json +37 -44
- package/templates/Dockerfile +32 -0
- package/templates/integration.ts +26 -5
- package/templates/openship.json +9 -0
- package/templates/railpack.json +1 -1
- package/templates/sandcms/blueprints/files/image.yaml +1 -1
- package/templates/sandcms/config.yaml +1 -1
- package/dist/cli/cloudflare.d.ts +0 -8
- package/dist/cli/cloudflare.js +0 -135
- package/dist/server/db/migrations/0001_independent_password_resets.sql +0 -3
- package/dist/server/db/migrations/0002_email_failure_reason.sql +0 -1
- package/dist/server/db/migrations/0003_direct_r2_uploads.sql +0 -10
- package/dist/server/db/migrations/0004_agent_api_keys.sql +0 -38
- package/dist/server/db/migrations/0005_api_retention_and_pagination.sql +0 -3
- package/dist/server/db/migrations/0006_durable_media_deletion.sql +0 -7
- package/dist/server/db/migrations/0007_content_and_job_indexes.sql +0 -2
- package/dist/server/db/migrations/0008_search_text_cache.sql +0 -45
- package/dist/server/db/migrations/meta/0001_snapshot.json +0 -1197
- package/dist/server/db/migrations/meta/0002_snapshot.json +0 -1204
- package/dist/server/db/migrations/meta/0003_snapshot.json +0 -1281
- package/dist/server/db/migrations/meta/0004_snapshot.json +0 -1546
- package/dist/server/db/migrations/meta/0005_snapshot.json +0 -1572
- package/dist/server/db/migrations/meta/0006_snapshot.json +0 -1627
- package/dist/server/db/migrations/meta/0007_snapshot.json +0 -1643
- package/dist/server/db/migrations/meta/0008_snapshot.json +0 -1711
- package/dist/worker/index.d.ts +0 -10
- package/dist/worker/index.js +0 -460
- package/dist/worker/local.d.ts +0 -5
- package/dist/worker/local.js +0 -57
- package/dist/worker/source.d.ts +0 -2
- package/dist/worker/source.js +0 -31
- package/dist/worker/uploads.d.ts +0 -22
- package/dist/worker/uploads.js +0 -48
- package/templates/sandcms/worker/index.ts +0 -1
- package/templates/sandcms/worker/local.ts +0 -1
- package/templates/sandcms/worker/wrangler.jsonc +0 -6
- /package/dist/{worker → server/media}/svg.d.ts +0 -0
- /package/dist/{worker → server/media}/svg.js +0 -0
|
@@ -1,7 +1,22 @@
|
|
|
1
|
+
import { type MediaProcessor, type MediaStorage, type UploadTicketResponse } from './contracts.js';
|
|
1
2
|
import type { Store } from '../db/store.js';
|
|
2
3
|
import type { Content } from '../content.js';
|
|
3
4
|
import type { Field, Payload, SiteDefinition } from '../../blueprints/types.js';
|
|
4
|
-
export
|
|
5
|
+
export interface MediaOptions {
|
|
6
|
+
secret: string;
|
|
7
|
+
/** Public origin; chunk URLs are absolute. */
|
|
8
|
+
origin: string;
|
|
9
|
+
mediaDir: string;
|
|
10
|
+
processor?: MediaProcessor;
|
|
11
|
+
storage?: MediaStorage;
|
|
12
|
+
/** Internal timings in ms, shortened by tests. */
|
|
13
|
+
timing?: {
|
|
14
|
+
heartbeat?: number;
|
|
15
|
+
chunkIdle?: number;
|
|
16
|
+
chunkDeadline?: number;
|
|
17
|
+
};
|
|
18
|
+
}
|
|
19
|
+
export declare function createMedia(store: Store, definition: SiteDefinition, content: Content, options: MediaOptions): {
|
|
5
20
|
get: (id: string) => {
|
|
6
21
|
blueprint: string;
|
|
7
22
|
metadata: Payload;
|
|
@@ -16,6 +31,9 @@ export declare function createMedia(store: Store, definition: SiteDefinition, co
|
|
|
16
31
|
error: string | null;
|
|
17
32
|
createdAt: number;
|
|
18
33
|
searchText: string | null;
|
|
34
|
+
width: number | null;
|
|
35
|
+
height: number | null;
|
|
36
|
+
uploadAttempt: string | null;
|
|
19
37
|
} | undefined;
|
|
20
38
|
upload: (file: File, blueprintId?: string) => Promise<{
|
|
21
39
|
blueprint: string;
|
|
@@ -31,18 +49,18 @@ export declare function createMedia(store: Store, definition: SiteDefinition, co
|
|
|
31
49
|
error: string | null;
|
|
32
50
|
createdAt: number;
|
|
33
51
|
searchText: string | null;
|
|
52
|
+
width: number | null;
|
|
53
|
+
height: number | null;
|
|
54
|
+
uploadAttempt: string | null;
|
|
34
55
|
}>;
|
|
35
56
|
initiateUpload: (input: {
|
|
36
57
|
name: string;
|
|
37
58
|
type: string;
|
|
38
59
|
size: number;
|
|
39
|
-
}, blueprintId
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
id: `${string}-${string}-${string}-${string}-${string}`;
|
|
44
|
-
}>;
|
|
45
|
-
completeUpload: (id: string) => {
|
|
60
|
+
}, blueprintId: string | undefined, principal: {
|
|
61
|
+
userId: string;
|
|
62
|
+
}) => Promise<UploadTicketResponse>;
|
|
63
|
+
completeUpload: (id: string) => Promise<{
|
|
46
64
|
blueprint: string;
|
|
47
65
|
metadata: Payload;
|
|
48
66
|
id: string;
|
|
@@ -56,9 +74,20 @@ export declare function createMedia(store: Store, definition: SiteDefinition, co
|
|
|
56
74
|
error: string | null;
|
|
57
75
|
createdAt: number;
|
|
58
76
|
searchText: string | null;
|
|
59
|
-
|
|
77
|
+
width: number | null;
|
|
78
|
+
height: number | null;
|
|
79
|
+
uploadAttempt: string | null;
|
|
80
|
+
}>;
|
|
60
81
|
tick: () => Promise<boolean>;
|
|
61
82
|
drain: () => Promise<void>;
|
|
83
|
+
receiveChunk: (id: string, index: number, body: ReadableStream<Uint8Array>, ticketToken: string | null, userId: string) => Promise<{
|
|
84
|
+
received: number[];
|
|
85
|
+
}>;
|
|
86
|
+
uploadStatus: (id: string, ticketToken: string | null, userId: string) => Promise<{
|
|
87
|
+
received: number[];
|
|
88
|
+
chunkSize: number;
|
|
89
|
+
complete: boolean;
|
|
90
|
+
}>;
|
|
62
91
|
retry: (id: string) => void;
|
|
63
92
|
version: string;
|
|
64
93
|
versionForAsset: (asset: NonNullable<ReturnType<(id: string) => {
|
|
@@ -75,8 +104,11 @@ export declare function createMedia(store: Store, definition: SiteDefinition, co
|
|
|
75
104
|
error: string | null;
|
|
76
105
|
createdAt: number;
|
|
77
106
|
searchText: string | null;
|
|
107
|
+
width: number | null;
|
|
108
|
+
height: number | null;
|
|
109
|
+
uploadAttempt: string | null;
|
|
78
110
|
} | undefined>>) => string;
|
|
79
|
-
|
|
111
|
+
storage: MediaStorage;
|
|
80
112
|
list: () => {
|
|
81
113
|
blueprint: string;
|
|
82
114
|
metadata: Payload;
|
|
@@ -91,10 +123,15 @@ export declare function createMedia(store: Store, definition: SiteDefinition, co
|
|
|
91
123
|
error: string | null;
|
|
92
124
|
createdAt: number;
|
|
93
125
|
searchText: string | null;
|
|
126
|
+
width: number | null;
|
|
127
|
+
height: number | null;
|
|
128
|
+
uploadAttempt: string | null;
|
|
94
129
|
}[];
|
|
95
130
|
updateMetadata: (id: string, input: unknown) => void;
|
|
96
131
|
health(): Promise<{
|
|
97
|
-
|
|
132
|
+
bytes: number;
|
|
133
|
+
free: number;
|
|
134
|
+
storage: "local";
|
|
98
135
|
}>;
|
|
99
136
|
rebuild: () => void;
|
|
100
137
|
remove(id: string): Promise<void>;
|
|
@@ -111,6 +148,7 @@ export declare function createMedia(store: Store, definition: SiteDefinition, co
|
|
|
111
148
|
width: number;
|
|
112
149
|
height: number;
|
|
113
150
|
format: string;
|
|
151
|
+
etag?: string;
|
|
114
152
|
}[];
|
|
115
153
|
photographer: {
|
|
116
154
|
id: string;
|
|
@@ -130,6 +168,7 @@ export declare function createMedia(store: Store, definition: SiteDefinition, co
|
|
|
130
168
|
width: number;
|
|
131
169
|
height: number;
|
|
132
170
|
format: string;
|
|
171
|
+
etag?: string;
|
|
133
172
|
}[];
|
|
134
173
|
photographer: {
|
|
135
174
|
id: string;
|
|
@@ -153,6 +192,7 @@ export declare function createMedia(store: Store, definition: SiteDefinition, co
|
|
|
153
192
|
width: number;
|
|
154
193
|
height: number;
|
|
155
194
|
format: string;
|
|
195
|
+
etag?: string;
|
|
156
196
|
}[];
|
|
157
197
|
photographer: {
|
|
158
198
|
id: string;
|
|
@@ -176,6 +216,7 @@ export declare function createMedia(store: Store, definition: SiteDefinition, co
|
|
|
176
216
|
width: number;
|
|
177
217
|
height: number;
|
|
178
218
|
format: string;
|
|
219
|
+
etag?: string;
|
|
179
220
|
}[];
|
|
180
221
|
photographer: {
|
|
181
222
|
id: string;
|
|
@@ -183,6 +224,7 @@ export declare function createMedia(store: Store, definition: SiteDefinition, co
|
|
|
183
224
|
} | null;
|
|
184
225
|
};
|
|
185
226
|
}[];
|
|
227
|
+
/** Background work: one render loop (one decode at a time) plus deletions, expiry and upload validation. */
|
|
186
228
|
start: () => () => void;
|
|
187
229
|
};
|
|
188
230
|
export type MediaService = ReturnType<typeof createMedia>;
|
|
@@ -1,22 +1,77 @@
|
|
|
1
|
+
import { closeSync, createReadStream, fstatSync, openSync } from 'node:fs';
|
|
2
|
+
import { Readable } from 'node:stream';
|
|
1
3
|
import { isLinkValue } from '../../link.js';
|
|
2
4
|
import { effectiveMetadata, fileFocus, getFileBlueprint, publicMetadata } from '../../blueprints/files.js';
|
|
3
5
|
import { validatePayload, walkValues } from '../../blueprints/validate.js';
|
|
4
6
|
import { mergeDraft } from '../../blueprints/merge.js';
|
|
5
|
-
import { createRunner } from './runner.js';
|
|
7
|
+
import { createRunner, liveUploadSession, processorQueue, variantKey } from './runner.js';
|
|
6
8
|
import { createDeletionRunner } from './deletions.js';
|
|
9
|
+
import { createLocalStorage } from './storage.js';
|
|
10
|
+
import { MediaRequestError, UPLOAD_CHUNK_SIZE, UPLOAD_TICKET_HEADER } from './contracts.js';
|
|
11
|
+
import { signUploadTicket, verifyUploadTicket } from './upload-ticket.js';
|
|
7
12
|
import { assetPresetVersion, presetVersion } from './presets.js';
|
|
8
|
-
import { and, eq, desc, isNotNull } from 'drizzle-orm';
|
|
13
|
+
import { and, eq, desc, inArray, isNotNull, sql } from 'drizzle-orm';
|
|
9
14
|
import { media, mediaDeletions, jobs, uploads, uploadSessions, references, documents } from '../db/schema.js';
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
15
|
+
const mimeTypes = [
|
|
16
|
+
'image/jpeg',
|
|
17
|
+
'image/png',
|
|
18
|
+
'image/webp',
|
|
19
|
+
'image/avif',
|
|
20
|
+
'image/gif',
|
|
21
|
+
'image/tiff',
|
|
22
|
+
'image/svg+xml'
|
|
23
|
+
];
|
|
24
|
+
const contentTypes = {
|
|
25
|
+
webp: 'image/webp',
|
|
26
|
+
avif: 'image/avif',
|
|
27
|
+
jpeg: 'image/jpeg',
|
|
28
|
+
svg: 'image/svg+xml'
|
|
29
|
+
};
|
|
30
|
+
const sessionTimeout = 15 * 60_000,
|
|
31
|
+
// A ticket outlives the idle timeout so a slow but active upload can finish.
|
|
32
|
+
ticketLifetime = 24 * 60 * 60_000, chunkLease = 120_000, diskMargin = 5_000_000_000;
|
|
33
|
+
/**
|
|
34
|
+
* Pass a request body through, failing with 408 when no bytes arrive for `idle` ms or the whole body takes
|
|
35
|
+
* longer than `deadline` ms, so a half-open connection cannot hold an upload lease forever.
|
|
36
|
+
*/
|
|
37
|
+
function guardStream(body, idle, deadline) {
|
|
38
|
+
const reader = body.getReader(), end = Date.now() + deadline;
|
|
39
|
+
return new ReadableStream({
|
|
40
|
+
async pull(controller) {
|
|
41
|
+
let timer;
|
|
42
|
+
const wait = Math.min(idle, end - Date.now());
|
|
43
|
+
const stalled = new Promise((_, reject) => {
|
|
44
|
+
timer = setTimeout(() => reject(new MediaRequestError(408, 'Chunk upload stalled. Send this chunk again.')), Math.max(0, wait));
|
|
45
|
+
});
|
|
46
|
+
try {
|
|
47
|
+
const { done, value } = await Promise.race([reader.read(), stalled]);
|
|
48
|
+
if (done)
|
|
49
|
+
controller.close();
|
|
50
|
+
else
|
|
51
|
+
controller.enqueue(value);
|
|
52
|
+
}
|
|
53
|
+
catch (error) {
|
|
54
|
+
void reader.cancel(error).catch(() => { });
|
|
55
|
+
controller.error(error);
|
|
56
|
+
}
|
|
57
|
+
finally {
|
|
58
|
+
clearTimeout(timer);
|
|
59
|
+
}
|
|
60
|
+
},
|
|
61
|
+
cancel: (reason) => reader.cancel(reason)
|
|
62
|
+
});
|
|
16
63
|
}
|
|
17
|
-
export function createMedia(store, definition, content,
|
|
64
|
+
export function createMedia(store, definition, content, options) {
|
|
18
65
|
const { db } = store, settings = definition.config.media;
|
|
19
66
|
const version = presetVersion(definition);
|
|
67
|
+
const storage = options.storage ?? createLocalStorage(options.mediaDir);
|
|
68
|
+
const origin = new URL(options.origin).origin;
|
|
69
|
+
let loaded;
|
|
70
|
+
// sharp is loaded only when an image is processed, so tests and CLI commands can inject a fake.
|
|
71
|
+
const loadProcessor = () => (loaded ??= options.processor
|
|
72
|
+
? Promise.resolve(options.processor)
|
|
73
|
+
: import('./processor.js').then((m) => m.createProcessor()));
|
|
74
|
+
const processor = processorQueue(loadProcessor);
|
|
20
75
|
const get = (id) => db.select().from(media).where(eq(media.id, id)).get();
|
|
21
76
|
// Only published snapshots grant access; draft edits never change public visibility.
|
|
22
77
|
const isPublic = (id) => !!db
|
|
@@ -25,18 +80,6 @@ export function createMedia(store, definition, content, token, workerUrl = defin
|
|
|
25
80
|
.innerJoin(documents, eq(documents.id, references.documentId))
|
|
26
81
|
.where(and(eq(references.targetId, id), eq(references.kind, 'media'), eq(references.snapshot, 'published'), isNotNull(documents.published)))
|
|
27
82
|
.get();
|
|
28
|
-
async function worker(path, init = {}) {
|
|
29
|
-
if (!token)
|
|
30
|
-
throw new Error('Configure SANDCMS_WORKER_TOKEN before uploading media');
|
|
31
|
-
const response = await fetch(workerUrl.replace(/\/$/, '') + '/internal/' + path, {
|
|
32
|
-
...init,
|
|
33
|
-
headers: { ...init.headers, authorization: `Bearer ${token}`, 'x-sandcms-protocol': '6' },
|
|
34
|
-
signal: init.signal ?? AbortSignal.timeout(60_000)
|
|
35
|
-
});
|
|
36
|
-
if (!response.ok && !(path.startsWith('media/') && response.status === 304))
|
|
37
|
-
throw new WorkerResponseError(response.status, (await response.text().catch(() => '')).slice(0, 300));
|
|
38
|
-
return response;
|
|
39
|
-
}
|
|
40
83
|
const assetVersion = (asset) => assetPresetVersion(definition, asset);
|
|
41
84
|
function enqueue(id, mime) {
|
|
42
85
|
const asset = get(id);
|
|
@@ -88,7 +131,7 @@ export function createMedia(store, definition, content, token, workerUrl = defin
|
|
|
88
131
|
.where(eq(media.id, id))
|
|
89
132
|
.run();
|
|
90
133
|
}
|
|
91
|
-
async function
|
|
134
|
+
async function createUpload(input, blueprintId) {
|
|
92
135
|
const blueprint = getFileBlueprint(definition, blueprintId);
|
|
93
136
|
if (!blueprint.accept.includes(input.type))
|
|
94
137
|
throw new Error(`Choose a file accepted by ${blueprint.label}`);
|
|
@@ -96,23 +139,23 @@ export function createMedia(store, definition, content, token, workerUrl = defin
|
|
|
96
139
|
throw new Error(`Choose an image under ${settings.uploadLimit / 1_000_000} MB`);
|
|
97
140
|
if (input.type === 'image/svg+xml' && input.size > 1_000_000)
|
|
98
141
|
throw new Error('Choose an SVG under 1 MB');
|
|
99
|
-
if (!
|
|
100
|
-
'image/jpeg',
|
|
101
|
-
'image/png',
|
|
102
|
-
'image/webp',
|
|
103
|
-
'image/avif',
|
|
104
|
-
'image/gif',
|
|
105
|
-
'image/svg+xml'
|
|
106
|
-
].includes(input.type))
|
|
142
|
+
if (!mimeTypes.includes(input.type))
|
|
107
143
|
throw new Error('Choose a raster image or SVG');
|
|
108
144
|
if (typeof input.name !== 'string' || !input.name.trim())
|
|
109
145
|
throw new Error('Choose a file');
|
|
110
|
-
|
|
111
|
-
const
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
146
|
+
// Reserve room for the chunks, the assembled file and its variants, plus a fixed margin.
|
|
147
|
+
const reserved = db
|
|
148
|
+
.select({ bytes: sql `coalesce(sum(${media.bytes}), 0)` })
|
|
149
|
+
.from(uploadSessions)
|
|
150
|
+
.innerJoin(media, eq(media.id, uploadSessions.mediaId))
|
|
151
|
+
.where(liveUploadSession())
|
|
152
|
+
.get()?.bytes ?? 0;
|
|
153
|
+
const usage = await storage.usage();
|
|
154
|
+
if (usage.free < 3 * input.size + reserved + diskMargin)
|
|
155
|
+
throw new MediaRequestError(507, 'Not enough disk space for this upload');
|
|
156
|
+
if (settings.maxStorage && usage.bytes + reserved + input.size > settings.maxStorage)
|
|
157
|
+
throw new MediaRequestError(413, 'Media storage quota exceeded');
|
|
158
|
+
const id = crypto.randomUUID(), attempt = crypto.randomUUID(), expiresAt = Date.now() + sessionTimeout;
|
|
116
159
|
db.transaction(() => {
|
|
117
160
|
db.insert(media)
|
|
118
161
|
.values({
|
|
@@ -126,11 +169,112 @@ export function createMedia(store, definition, content, token, workerUrl = defin
|
|
|
126
169
|
createdAt: Date.now()
|
|
127
170
|
})
|
|
128
171
|
.run();
|
|
129
|
-
db.insert(uploadSessions).values({ mediaId: id, attempt, expiresAt
|
|
172
|
+
db.insert(uploadSessions).values({ mediaId: id, attempt, expiresAt }).run();
|
|
173
|
+
});
|
|
174
|
+
return { id, attempt, expiresAt };
|
|
175
|
+
}
|
|
176
|
+
async function initiateUpload(input, blueprintId, principal) {
|
|
177
|
+
const { id, attempt, expiresAt } = await createUpload(input, blueprintId);
|
|
178
|
+
const ticket = signUploadTicket(options.secret, {
|
|
179
|
+
id,
|
|
180
|
+
attempt,
|
|
181
|
+
mime: input.type,
|
|
182
|
+
bytes: input.size,
|
|
183
|
+
sub: principal.userId,
|
|
184
|
+
exp: Date.now() + ticketLifetime
|
|
185
|
+
});
|
|
186
|
+
return {
|
|
187
|
+
id,
|
|
188
|
+
url: `${origin}/api/cms/media/upload/${id}`,
|
|
189
|
+
chunkSize: UPLOAD_CHUNK_SIZE,
|
|
190
|
+
headers: { [UPLOAD_TICKET_HEADER]: ticket },
|
|
191
|
+
expiresAt
|
|
192
|
+
};
|
|
193
|
+
}
|
|
194
|
+
function authorize(id, token, userId) {
|
|
195
|
+
const ticket = verifyUploadTicket(options.secret, token);
|
|
196
|
+
if (!ticket || ticket.id !== id || ticket.sub !== userId)
|
|
197
|
+
throw new MediaRequestError(403, 'Upload ticket is invalid or expired');
|
|
198
|
+
return ticket;
|
|
199
|
+
}
|
|
200
|
+
// A chunk holds the session lease while it streams and slides the idle expiry forward.
|
|
201
|
+
const streaming = new Map();
|
|
202
|
+
const holdSession = (id, attempt, release = false) => {
|
|
203
|
+
const now = Date.now();
|
|
204
|
+
db.update(uploadSessions)
|
|
205
|
+
.set({
|
|
206
|
+
status: 'receiving',
|
|
207
|
+
leaseUntil: release ? 0 : now + chunkLease,
|
|
208
|
+
expiresAt: sql `max(${uploadSessions.expiresAt}, ${now + sessionTimeout})`
|
|
209
|
+
})
|
|
210
|
+
.where(and(eq(uploadSessions.mediaId, id), eq(uploadSessions.attempt, attempt), inArray(uploadSessions.status, ['uploading', 'receiving'])))
|
|
211
|
+
.run();
|
|
212
|
+
};
|
|
213
|
+
function openSession(id, attempt) {
|
|
214
|
+
const asset = get(id);
|
|
215
|
+
const session = db.select().from(uploadSessions).where(eq(uploadSessions.mediaId, id)).get();
|
|
216
|
+
if (!asset || !session || session.attempt !== attempt)
|
|
217
|
+
throw new MediaRequestError(404, 'Upload not found. Upload this file again.');
|
|
218
|
+
if (asset.status === 'deleting' ||
|
|
219
|
+
db.select().from(mediaDeletions).where(eq(mediaDeletions.mediaId, id)).get())
|
|
220
|
+
throw new MediaRequestError(409, 'This image is being deleted');
|
|
221
|
+
if (session.status === 'expired')
|
|
222
|
+
throw new MediaRequestError(410, 'Upload expired. Upload this file again.');
|
|
223
|
+
return { asset, session };
|
|
224
|
+
}
|
|
225
|
+
async function writeChunk(id, attempt, bytes, index, body) {
|
|
226
|
+
const count = Math.ceil(bytes / UPLOAD_CHUNK_SIZE);
|
|
227
|
+
if (!Number.isSafeInteger(index) || index < 0 || index >= count)
|
|
228
|
+
throw new MediaRequestError(400, 'Invalid chunk index');
|
|
229
|
+
const expected = index === count - 1 ? bytes - index * UPLOAD_CHUNK_SIZE : UPLOAD_CHUNK_SIZE;
|
|
230
|
+
db.transaction(() => {
|
|
231
|
+
const { session } = openSession(id, attempt);
|
|
232
|
+
const now = Date.now();
|
|
233
|
+
if (session.status !== 'uploading' && session.status !== 'receiving')
|
|
234
|
+
throw new MediaRequestError(409, 'This upload is already complete');
|
|
235
|
+
if (session.expiresAt <= now && session.leaseUntil <= now)
|
|
236
|
+
throw new MediaRequestError(410, 'Upload expired. Upload this file again.');
|
|
237
|
+
holdSession(id, attempt);
|
|
130
238
|
});
|
|
131
|
-
|
|
239
|
+
streaming.set(id, (streaming.get(id) ?? 0) + 1);
|
|
240
|
+
const keepAlive = setInterval(() => holdSession(id, attempt), chunkLease / 4);
|
|
241
|
+
try {
|
|
242
|
+
await storage.writeChunk(id, attempt, index, guardStream(body, options.timing?.chunkIdle ?? 60_000, options.timing?.chunkDeadline ?? 15 * 60_000), expected, true);
|
|
243
|
+
}
|
|
244
|
+
finally {
|
|
245
|
+
clearInterval(keepAlive);
|
|
246
|
+
const left = streaming.get(id) - 1;
|
|
247
|
+
if (left)
|
|
248
|
+
streaming.set(id, left);
|
|
249
|
+
else
|
|
250
|
+
streaming.delete(id);
|
|
251
|
+
// Nothing is in flight once the last chunk lands; the sliding expiry covers idle time.
|
|
252
|
+
holdSession(id, attempt, !left);
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
async function receiveChunk(id, index, body, ticketToken, userId) {
|
|
256
|
+
const ticket = authorize(id, ticketToken, userId);
|
|
257
|
+
await writeChunk(id, ticket.attempt, ticket.bytes, index, body);
|
|
258
|
+
return { received: await storage.listChunks(id, ticket.attempt) };
|
|
259
|
+
}
|
|
260
|
+
/** Stored chunks for resume; `complete` once the chunks were assembled (e.g. the completion response was lost). */
|
|
261
|
+
async function uploadStatus(id, ticketToken, userId) {
|
|
262
|
+
const ticket = authorize(id, ticketToken, userId);
|
|
263
|
+
const asset = get(id), session = db.select().from(uploadSessions).where(eq(uploadSessions.mediaId, id)).get();
|
|
264
|
+
if (asset &&
|
|
265
|
+
asset.status !== 'deleting' &&
|
|
266
|
+
(session
|
|
267
|
+
? session.attempt === ticket.attempt && session.status === 'pending'
|
|
268
|
+
: asset.uploadAttempt === ticket.attempt))
|
|
269
|
+
return { received: [], chunkSize: UPLOAD_CHUNK_SIZE, complete: true };
|
|
270
|
+
openSession(id, ticket.attempt);
|
|
271
|
+
return {
|
|
272
|
+
received: await storage.listChunks(id, ticket.attempt),
|
|
273
|
+
chunkSize: UPLOAD_CHUNK_SIZE,
|
|
274
|
+
complete: false
|
|
275
|
+
};
|
|
132
276
|
}
|
|
133
|
-
function completeUpload(id) {
|
|
277
|
+
async function completeUpload(id) {
|
|
134
278
|
const asset = get(id);
|
|
135
279
|
if (!asset)
|
|
136
280
|
throw new Error('Image not found');
|
|
@@ -145,38 +289,64 @@ export function createMedia(store, definition, content, token, workerUrl = defin
|
|
|
145
289
|
}
|
|
146
290
|
if (session.status === 'expired')
|
|
147
291
|
throw new Error('Upload expired. Upload this file again.');
|
|
148
|
-
if (session.status === '
|
|
149
|
-
|
|
150
|
-
|
|
292
|
+
if (session.status === 'pending')
|
|
293
|
+
return asset;
|
|
294
|
+
// Hold the lease while assembling so neither expiry nor deletion removes the chunks.
|
|
295
|
+
const claimed = db.transaction(() => {
|
|
296
|
+
const current = db.select().from(uploadSessions).where(eq(uploadSessions.mediaId, id)).get();
|
|
297
|
+
const now = Date.now();
|
|
298
|
+
if (!current || current.attempt !== session.attempt)
|
|
299
|
+
return 'gone';
|
|
300
|
+
if (current.status === 'pending')
|
|
301
|
+
return 'done';
|
|
302
|
+
if (current.status === 'expired' || (current.expiresAt < now && current.leaseUntil <= now))
|
|
303
|
+
return 'expired';
|
|
151
304
|
db.update(uploadSessions)
|
|
152
|
-
.set({
|
|
305
|
+
.set({ leaseUntil: now + chunkLease })
|
|
153
306
|
.where(eq(uploadSessions.mediaId, id))
|
|
154
307
|
.run();
|
|
308
|
+
return 'claimed';
|
|
309
|
+
});
|
|
310
|
+
if (claimed === 'done')
|
|
311
|
+
return asset;
|
|
312
|
+
if (claimed === 'gone')
|
|
313
|
+
throw new Error('Upload was interrupted. Upload this file again.');
|
|
314
|
+
if (claimed === 'expired')
|
|
315
|
+
throw new Error('Upload expired. Upload this file again.');
|
|
316
|
+
try {
|
|
317
|
+
await storage.assemble(id, session.attempt, asset.bytes);
|
|
318
|
+
}
|
|
319
|
+
catch (e) {
|
|
320
|
+
db.update(uploadSessions).set({ leaseUntil: 0 }).where(eq(uploadSessions.mediaId, id)).run();
|
|
321
|
+
throw e;
|
|
155
322
|
}
|
|
323
|
+
db.update(uploadSessions)
|
|
324
|
+
.set({ status: 'pending', leaseUntil: 0 })
|
|
325
|
+
.where(and(eq(uploadSessions.mediaId, id), eq(uploadSessions.attempt, session.attempt), inArray(uploadSessions.status, ['uploading', 'receiving'])))
|
|
326
|
+
.run();
|
|
156
327
|
return asset;
|
|
157
328
|
}
|
|
158
|
-
// Server callers
|
|
329
|
+
// Server callers stage the bytes directly and share the browser's durable finalization.
|
|
159
330
|
async function upload(file, blueprintId) {
|
|
160
|
-
const
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
body: file,
|
|
165
|
-
signal: AbortSignal.timeout(120_000)
|
|
166
|
-
});
|
|
167
|
-
if (!response.ok)
|
|
168
|
-
throw new Error('Direct upload failed. Upload this file again.');
|
|
169
|
-
return completeUpload(signed.id);
|
|
331
|
+
const { id, attempt } = await createUpload(file, blueprintId);
|
|
332
|
+
for (let index = 0; index * UPLOAD_CHUNK_SIZE < file.size; index++)
|
|
333
|
+
await writeChunk(id, attempt, file.size, index, file.slice(index * UPLOAD_CHUNK_SIZE, (index + 1) * UPLOAD_CHUNK_SIZE).stream());
|
|
334
|
+
return completeUpload(id);
|
|
170
335
|
}
|
|
171
|
-
const deletePending = createDeletionRunner(store,
|
|
172
|
-
const
|
|
173
|
-
|
|
174
|
-
|
|
336
|
+
const deletePending = createDeletionRunner(store, storage);
|
|
337
|
+
const runner = createRunner(store, {
|
|
338
|
+
storage,
|
|
339
|
+
processor,
|
|
340
|
+
maxPixels: settings.maxPixels,
|
|
341
|
+
owner: crypto.randomUUID(),
|
|
342
|
+
heartbeat: options.timing?.heartbeat,
|
|
343
|
+
enqueue,
|
|
344
|
+
deletePending
|
|
345
|
+
});
|
|
346
|
+
const tick = async () => (await runner.maintain()) || (await runner.render());
|
|
175
347
|
async function drain() {
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
;
|
|
179
|
-
}));
|
|
348
|
+
while (await tick())
|
|
349
|
+
;
|
|
180
350
|
}
|
|
181
351
|
function retry(id) {
|
|
182
352
|
const asset = get(id);
|
|
@@ -303,14 +473,16 @@ export function createMedia(store, definition, content, token, workerUrl = defin
|
|
|
303
473
|
completeUpload,
|
|
304
474
|
tick,
|
|
305
475
|
drain,
|
|
476
|
+
receiveChunk,
|
|
477
|
+
uploadStatus,
|
|
306
478
|
retry,
|
|
307
479
|
version,
|
|
308
480
|
versionForAsset: assetVersion,
|
|
309
|
-
|
|
481
|
+
storage,
|
|
310
482
|
list: () => db.select().from(media).orderBy(desc(media.createdAt)).all(),
|
|
311
483
|
updateMetadata,
|
|
312
484
|
async health() {
|
|
313
|
-
return (await
|
|
485
|
+
return { storage: 'local', ...(await storage.usage()) };
|
|
314
486
|
},
|
|
315
487
|
rebuild: () => {
|
|
316
488
|
db.transaction(() => {
|
|
@@ -347,10 +519,7 @@ export function createMedia(store, definition, content, token, workerUrl = defin
|
|
|
347
519
|
// Persist intent before awaiting storage; processors retain their leases until finished.
|
|
348
520
|
db.transaction(() => {
|
|
349
521
|
content.clearReferences(id, 'media');
|
|
350
|
-
db.insert(mediaDeletions)
|
|
351
|
-
.values({ mediaId: id })
|
|
352
|
-
.onConflictDoNothing()
|
|
353
|
-
.run();
|
|
522
|
+
db.insert(mediaDeletions).values({ mediaId: id }).onConflictDoNothing().run();
|
|
354
523
|
db.update(media)
|
|
355
524
|
.set({
|
|
356
525
|
status: 'deleting',
|
|
@@ -367,59 +536,99 @@ export function createMedia(store, definition, content, token, workerUrl = defin
|
|
|
367
536
|
// Metadata only. Bytes still require an authenticated CMS session.
|
|
368
537
|
previewAsset: (id, overrides = {}) => assetData(id, overrides, true),
|
|
369
538
|
async deliver(key, method, preview = false, ifNoneMatch = null) {
|
|
539
|
+
const notFound = () => new Response('Not found', { status: 404 });
|
|
370
540
|
const asset = get(key.split('/')[0]);
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
541
|
+
const variant = asset?.activeVersion ? asset.variants.find((v) => v.key === key) : undefined;
|
|
542
|
+
if (!asset || !variant || (!preview && !isPublic(asset.id)))
|
|
543
|
+
return notFound();
|
|
544
|
+
// Rebuilds re-render live keys: a job that is not ready means the file is changing, so send no validator.
|
|
545
|
+
const version = key.split('/')[1];
|
|
546
|
+
const job = db
|
|
547
|
+
.select()
|
|
548
|
+
.from(jobs)
|
|
549
|
+
.where(and(eq(jobs.mediaId, asset.id), eq(jobs.version, version)))
|
|
550
|
+
.all()
|
|
551
|
+
.find((j) => (j.result?.key ?? variantKey(asset.id, j.version, j.spec)) === key);
|
|
552
|
+
const tag = job ? (job.status === 'ready' ? job.result?.etag : undefined) : variant.etag;
|
|
553
|
+
const path = storage.variantPath(key);
|
|
554
|
+
// Size and bytes come from one open file, even if a rebuild replaces the path meanwhile.
|
|
555
|
+
let fd;
|
|
375
556
|
try {
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
});
|
|
380
|
-
// Publication can change while the remote read is in flight.
|
|
381
|
-
const current = get(asset.id);
|
|
382
|
-
if (!current?.variants.some((v) => v.key === key) || (!preview && !isPublic(asset.id))) {
|
|
383
|
-
await response.body?.cancel();
|
|
384
|
-
return new Response('Not found', { status: 404 });
|
|
385
|
-
}
|
|
386
|
-
return response;
|
|
557
|
+
if (!path)
|
|
558
|
+
return notFound();
|
|
559
|
+
fd = openSync(path, 'r');
|
|
387
560
|
}
|
|
388
|
-
catch
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
561
|
+
catch {
|
|
562
|
+
return notFound();
|
|
563
|
+
}
|
|
564
|
+
let size;
|
|
565
|
+
try {
|
|
566
|
+
size = fstatSync(fd).size;
|
|
392
567
|
}
|
|
568
|
+
catch {
|
|
569
|
+
closeSync(fd);
|
|
570
|
+
return notFound();
|
|
571
|
+
}
|
|
572
|
+
const headers = new Headers({
|
|
573
|
+
'Content-Type': contentTypes[variant.format] ?? 'application/octet-stream',
|
|
574
|
+
'Cache-Control': 'private, no-store',
|
|
575
|
+
'Content-Security-Policy': "default-src 'none'; style-src 'none'; sandbox",
|
|
576
|
+
'Cross-Origin-Resource-Policy': 'cross-origin',
|
|
577
|
+
'X-Content-Type-Options': 'nosniff'
|
|
578
|
+
});
|
|
579
|
+
if (tag)
|
|
580
|
+
headers.set('ETag', `"${tag}"`);
|
|
581
|
+
// GET/HEAD use weak comparison; a validator list may include quoted commas.
|
|
582
|
+
if (ifNoneMatch?.trim() === '*' ||
|
|
583
|
+
(tag &&
|
|
584
|
+
Array.from(ifNoneMatch?.matchAll(/(?:W\/)?"[^"]*"/g) ?? []).some(([value]) => value.replace(/^W\//, '') === `"${tag}"`))) {
|
|
585
|
+
closeSync(fd);
|
|
586
|
+
return new Response(null, { status: 304, headers });
|
|
587
|
+
}
|
|
588
|
+
headers.set('Content-Length', String(size));
|
|
589
|
+
if (method === 'HEAD' || !size) {
|
|
590
|
+
closeSync(fd);
|
|
591
|
+
return new Response(null, { headers });
|
|
592
|
+
}
|
|
593
|
+
const body = Readable.toWeb(createReadStream('', { fd, start: 0, end: size - 1, autoClose: true }));
|
|
594
|
+
return new Response(body, { headers });
|
|
393
595
|
},
|
|
394
596
|
publicGallery: (rows, field) => galleryData(rows, field),
|
|
395
597
|
previewGallery: (rows, field) => galleryData(rows, field, true),
|
|
598
|
+
/** Background work: one render loop (one decode at a time) plus deletions, expiry and upload validation. */
|
|
396
599
|
start: () => {
|
|
397
|
-
let
|
|
398
|
-
|
|
399
|
-
async function loop() {
|
|
400
|
-
if (stopped)
|
|
600
|
+
let stopped = false, swept = 0;
|
|
601
|
+
const running = new Set();
|
|
602
|
+
async function loop(name, step) {
|
|
603
|
+
if (stopped || running.has(name))
|
|
401
604
|
return;
|
|
402
|
-
|
|
605
|
+
running.add(name);
|
|
403
606
|
try {
|
|
404
|
-
while (!stopped && (await
|
|
405
|
-
|
|
406
|
-
void loop();
|
|
607
|
+
while (!stopped && (await step()))
|
|
608
|
+
;
|
|
407
609
|
}
|
|
408
610
|
catch (e) {
|
|
409
611
|
console.error('SandCMS media job failed', e);
|
|
410
612
|
}
|
|
411
613
|
finally {
|
|
412
|
-
|
|
614
|
+
running.delete(name);
|
|
413
615
|
}
|
|
414
616
|
}
|
|
415
617
|
const timer = setInterval(() => {
|
|
416
|
-
|
|
417
|
-
|
|
618
|
+
void loop('maintain', runner.maintain);
|
|
619
|
+
void loop('render', runner.render);
|
|
620
|
+
if (Date.now() - swept < 3_600_000)
|
|
621
|
+
return;
|
|
622
|
+
swept = Date.now();
|
|
623
|
+
storage
|
|
624
|
+
.sweepTmp(new Date(swept - 24 * 3_600_000))
|
|
625
|
+
.catch((e) => console.error('SandCMS media cleanup failed', e));
|
|
418
626
|
}, 1000);
|
|
419
627
|
timer.unref();
|
|
420
628
|
return () => {
|
|
421
629
|
stopped = true;
|
|
422
630
|
clearInterval(timer);
|
|
631
|
+
void loaded?.then((p) => p.close()).catch(() => { });
|
|
423
632
|
};
|
|
424
633
|
}
|
|
425
634
|
};
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
import { type MediaStorage } from './contracts.js';
|
|
2
|
+
/** Same shape as the public delivery route: `{id}/{version}/{preset}-{size}.{format}`. */
|
|
3
|
+
export declare const variantKeyPattern: RegExp;
|
|
4
|
+
/** Local-disk media storage. See contracts.ts for the layout and durability rules. */
|
|
5
|
+
export declare function createLocalStorage(root: string): MediaStorage;
|