wrangler 0.0.0-e33bea9b → 0.0.0-e351afcf
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/miniflare-dist/index.mjs +167 -93
- package/package.json +14 -9
- package/src/__tests__/api-devregistry.test.ts +121 -0
- package/src/__tests__/api.test.ts +27 -5
- package/src/__tests__/configuration.test.ts +59 -12
- package/src/__tests__/constellation.test.ts +371 -0
- package/src/__tests__/d1/execute.test.ts +2 -0
- package/src/__tests__/d1/migrate.test.ts +102 -0
- package/src/__tests__/delete.test.ts +6 -0
- package/src/__tests__/deployments.test.ts +335 -95
- package/src/__tests__/dev.test.tsx +71 -56
- package/src/__tests__/generate.test.ts +3 -4
- package/src/__tests__/helpers/mock-console.ts +6 -0
- package/src/__tests__/helpers/msw/handlers/deployments.ts +70 -3
- package/src/__tests__/helpers/msw/index.ts +4 -2
- package/src/__tests__/helpers/worker-scripts/parent-worker.js +4 -1
- package/src/__tests__/index.test.ts +14 -6
- package/src/__tests__/init.test.ts +127 -96
- package/src/__tests__/jest.setup.ts +4 -0
- package/src/__tests__/kv.test.ts +9 -9
- package/src/__tests__/logger.test.ts +68 -2
- package/src/__tests__/middleware.scheduled.test.ts +2 -2
- package/src/__tests__/middleware.test.ts +2 -2
- package/src/__tests__/mtls-certificates.test.ts +7 -3
- package/src/__tests__/pages/deployment-list.test.ts +2 -0
- package/src/__tests__/pages/functions-build.test.ts +70 -0
- package/src/__tests__/pages/project-list.test.ts +2 -0
- package/src/__tests__/pages/project-upload.test.ts +43 -24
- package/src/__tests__/pages/publish.test.ts +136 -92
- package/src/__tests__/publish.test.ts +878 -332
- package/src/__tests__/pubsub.test.ts +3 -0
- package/src/__tests__/queues.test.ts +5 -2
- package/src/__tests__/traverse-module-graph.test.ts +220 -0
- package/src/__tests__/tsconfig.tsbuildinfo +1 -1
- package/src/__tests__/user.test.ts +1 -1
- package/src/__tests__/worker-namespace.test.ts +1 -0
- package/src/api/dev.ts +12 -18
- package/src/api/pages/create-worker-bundle-contents.ts +1 -0
- package/src/api/pages/publish.tsx +28 -12
- package/src/bundle.ts +26 -9
- package/src/config/environment.ts +27 -0
- package/src/config/index.ts +18 -0
- package/src/config/validation.ts +91 -1
- package/src/constellation/createProject.tsx +51 -0
- package/src/constellation/deleteProject.ts +51 -0
- package/src/constellation/deleteProjectModel.ts +68 -0
- package/src/constellation/index.ts +75 -0
- package/src/constellation/listCatalog.tsx +35 -0
- package/src/constellation/listModel.tsx +41 -0
- package/src/constellation/listProject.tsx +28 -0
- package/src/constellation/listRuntime.tsx +28 -0
- package/src/constellation/options.ts +17 -0
- package/src/constellation/types.ts +17 -0
- package/src/constellation/uploadModel.tsx +64 -0
- package/src/constellation/utils.ts +90 -0
- package/src/create-worker-upload-form.ts +18 -1
- package/src/d1/backups.tsx +19 -15
- package/src/d1/create.tsx +45 -39
- package/src/d1/delete.ts +1 -1
- package/src/d1/execute.tsx +23 -20
- package/src/d1/list.tsx +14 -12
- package/src/d1/migrations/apply.tsx +36 -30
- package/src/d1/migrations/create.tsx +15 -12
- package/src/d1/migrations/helpers.ts +5 -8
- package/src/d1/migrations/list.tsx +10 -7
- package/src/deployments.ts +260 -8
- package/src/dev/dev.tsx +2 -0
- package/src/dev/local.tsx +48 -20
- package/src/dev/start-server.ts +49 -46
- package/src/dev/use-esbuild.ts +52 -49
- package/src/dev-registry.ts +2 -1
- package/src/dev.tsx +11 -5
- package/src/docs/helpers.ts +50 -0
- package/src/docs/index.ts +25 -68
- package/src/entry.ts +19 -8
- package/src/environment-variables/factory.ts +2 -1
- package/src/generate/index.ts +2 -2
- package/src/index.ts +86 -23
- package/src/init.ts +144 -135
- package/src/kv/helpers.ts +7 -4
- package/src/logger.ts +23 -4
- package/src/metrics/send-event.ts +2 -1
- package/src/miniflare-cli/index.ts +2 -2
- package/src/miniflare-cli/tsconfig.tsbuildinfo +1 -1
- package/src/module-collection.ts +92 -26
- package/src/pages/build.ts +29 -17
- package/src/pages/deployments.tsx +3 -5
- package/src/pages/dev.ts +24 -6
- package/src/pages/functions/buildPlugin.ts +1 -0
- package/src/pages/functions/buildWorker.ts +59 -8
- package/src/pages/functions/tsconfig.tsbuildinfo +1 -1
- package/src/pages/projects.tsx +2 -5
- package/src/pages/upload.tsx +24 -4
- package/src/publish/publish.ts +10 -26
- package/src/queues/cli/commands/consumer/add.ts +6 -0
- package/src/queues/client.ts +1 -0
- package/src/secret/index.ts +1 -0
- package/src/sites.ts +164 -52
- package/src/traverse-module-graph.ts +53 -0
- package/src/user/user.ts +1 -0
- package/src/utils/render.ts +93 -0
- package/src/worker.ts +10 -0
- package/wrangler-dist/cli.d.ts +29 -2
- package/wrangler-dist/cli.js +23270 -22110
- package/src/__tests__/api-devregistry.test.js +0 -64
package/src/pages/projects.tsx
CHANGED
|
@@ -1,5 +1,4 @@
|
|
|
1
1
|
import { execSync } from "node:child_process";
|
|
2
|
-
import { render } from "ink";
|
|
3
2
|
import Table from "ink-table";
|
|
4
3
|
import React from "react";
|
|
5
4
|
import { format as timeagoFormat } from "timeago.js";
|
|
@@ -10,6 +9,7 @@ import { FatalError } from "../errors";
|
|
|
10
9
|
import { logger } from "../logger";
|
|
11
10
|
import * as metrics from "../metrics";
|
|
12
11
|
import { requireAuth } from "../user";
|
|
12
|
+
import { renderToString } from "../utils/render";
|
|
13
13
|
import { PAGES_CONFIG_CACHE_FILENAME } from "./constants";
|
|
14
14
|
import { pagesBetaWarning } from "./utils";
|
|
15
15
|
import type {
|
|
@@ -44,10 +44,7 @@ export async function ListHandler() {
|
|
|
44
44
|
account_id: accountId,
|
|
45
45
|
});
|
|
46
46
|
|
|
47
|
-
|
|
48
|
-
patchConsole: false,
|
|
49
|
-
});
|
|
50
|
-
unmount();
|
|
47
|
+
logger.log(renderToString(<Table data={data}></Table>));
|
|
51
48
|
await metrics.sendMetricsEvent("list pages projects");
|
|
52
49
|
}
|
|
53
50
|
|
package/src/pages/upload.tsx
CHANGED
|
@@ -276,9 +276,7 @@ export const upload = async (
|
|
|
276
276
|
}
|
|
277
277
|
|
|
278
278
|
let counter = fileMap.size - sortedFiles.length;
|
|
279
|
-
const { rerender, unmount } =
|
|
280
|
-
<Progress done={counter} total={fileMap.size} />
|
|
281
|
-
);
|
|
279
|
+
const { rerender, unmount } = renderProgress(counter, fileMap.size);
|
|
282
280
|
|
|
283
281
|
const queue = new PQueue({ concurrency: BULK_UPLOAD_CONCURRENCY });
|
|
284
282
|
|
|
@@ -336,7 +334,7 @@ export const upload = async (
|
|
|
336
334
|
doUpload().then(
|
|
337
335
|
() => {
|
|
338
336
|
counter += bucket.files.length;
|
|
339
|
-
rerender(
|
|
337
|
+
rerender(counter, fileMap.size);
|
|
340
338
|
},
|
|
341
339
|
(error) => {
|
|
342
340
|
return Promise.reject(
|
|
@@ -437,6 +435,28 @@ function formatTime(duration: number) {
|
|
|
437
435
|
return `(${(duration / 1000).toFixed(2)} sec)`;
|
|
438
436
|
}
|
|
439
437
|
|
|
438
|
+
function renderProgress(done: number, total: number) {
|
|
439
|
+
if (isInteractive()) {
|
|
440
|
+
const { rerender, unmount } = render(
|
|
441
|
+
<Progress done={done} total={total} />
|
|
442
|
+
);
|
|
443
|
+
return {
|
|
444
|
+
// eslint-disable-next-line no-shadow
|
|
445
|
+
rerender(done: number, total: number) {
|
|
446
|
+
rerender(<Progress done={done} total={total} />);
|
|
447
|
+
},
|
|
448
|
+
unmount,
|
|
449
|
+
};
|
|
450
|
+
} else {
|
|
451
|
+
// eslint-disable-next-line no-shadow
|
|
452
|
+
const rerender = (done: number, total: number) => {
|
|
453
|
+
logger.log(`Uploading... (${done}/${total})`);
|
|
454
|
+
};
|
|
455
|
+
rerender(done, total);
|
|
456
|
+
return { rerender, unmount() {} };
|
|
457
|
+
}
|
|
458
|
+
}
|
|
459
|
+
|
|
440
460
|
function Progress({ done, total }: { done: number; total: number }) {
|
|
441
461
|
return (
|
|
442
462
|
<>
|
package/src/publish/publish.ts
CHANGED
|
@@ -12,6 +12,7 @@ import {
|
|
|
12
12
|
import { fetchListResult, fetchResult } from "../cfetch";
|
|
13
13
|
import { printBindings } from "../config";
|
|
14
14
|
import { createWorkerUploadForm } from "../create-worker-upload-form";
|
|
15
|
+
import { addHyphens } from "../deployments";
|
|
15
16
|
import { confirm } from "../dialogs";
|
|
16
17
|
import { getMigrationsToUpload } from "../durable";
|
|
17
18
|
import { logger } from "../logger";
|
|
@@ -20,6 +21,7 @@ import { ParseError } from "../parse";
|
|
|
20
21
|
import { getQueue, putConsumer } from "../queues/client";
|
|
21
22
|
import { getWorkersDevSubdomain } from "../routes";
|
|
22
23
|
import { syncAssets } from "../sites";
|
|
24
|
+
import traverseModuleGraph from "../traverse-module-graph";
|
|
23
25
|
import { identifyD1BindingsAsBeta } from "../worker";
|
|
24
26
|
import { getZoneForRoute } from "../zones";
|
|
25
27
|
import type { FetchError } from "../cfetch";
|
|
@@ -30,7 +32,6 @@ import type {
|
|
|
30
32
|
ZoneNameRoute,
|
|
31
33
|
CustomDomainRoute,
|
|
32
34
|
} from "../config/environment";
|
|
33
|
-
import type { DeploymentListResult } from "../deployments";
|
|
34
35
|
import type { Entry } from "../entry";
|
|
35
36
|
import type { PutConsumerBody } from "../queues/client";
|
|
36
37
|
import type { AssetPaths } from "../sites";
|
|
@@ -335,7 +336,7 @@ See https://developers.cloudflare.com/workers/platform/compatibility-dates for m
|
|
|
335
336
|
|
|
336
337
|
const jsxFactory = props.jsxFactory || config.jsx_factory;
|
|
337
338
|
const jsxFragment = props.jsxFragment || config.jsx_fragment;
|
|
338
|
-
const keepVars = props.keepVars
|
|
339
|
+
const keepVars = props.keepVars || config.keep_vars;
|
|
339
340
|
|
|
340
341
|
const minify = props.minify ?? config.minify;
|
|
341
342
|
|
|
@@ -401,7 +402,7 @@ See https://developers.cloudflare.com/workers/platform/compatibility-dates for m
|
|
|
401
402
|
: `/accounts/${accountId}/workers/scripts/${scriptName}`;
|
|
402
403
|
|
|
403
404
|
let available_on_subdomain: boolean | undefined = undefined; // we'll set this later
|
|
404
|
-
let
|
|
405
|
+
let deploymentId: string | null = null;
|
|
405
406
|
|
|
406
407
|
const { format } = props.entry;
|
|
407
408
|
|
|
@@ -463,14 +464,7 @@ See https://developers.cloudflare.com/workers/platform/compatibility-dates for m
|
|
|
463
464
|
resolvedEntryPointPath,
|
|
464
465
|
bundleType,
|
|
465
466
|
}: Awaited<ReturnType<typeof bundleWorker>> = props.noBundle
|
|
466
|
-
?
|
|
467
|
-
{
|
|
468
|
-
modules: [],
|
|
469
|
-
dependencies: {},
|
|
470
|
-
resolvedEntryPointPath: props.entry.file,
|
|
471
|
-
bundleType: props.entry.format === "modules" ? "esm" : "commonjs",
|
|
472
|
-
stop: undefined,
|
|
473
|
-
}
|
|
467
|
+
? await traverseModuleGraph(props.entry, props.rules)
|
|
474
468
|
: await bundleWorker(
|
|
475
469
|
props.entry,
|
|
476
470
|
typeof destination === "string" ? destination : destination.path,
|
|
@@ -541,6 +535,7 @@ See https://developers.cloudflare.com/workers/platform/compatibility-dates for m
|
|
|
541
535
|
? { binding: "__STATIC_CONTENT", id: assets.namespace }
|
|
542
536
|
: []
|
|
543
537
|
),
|
|
538
|
+
send_email: config.send_email,
|
|
544
539
|
vars: { ...config.vars, ...props.vars },
|
|
545
540
|
wasm_modules: config.wasm_modules,
|
|
546
541
|
text_blobs: {
|
|
@@ -632,7 +627,7 @@ See https://developers.cloudflare.com/workers/platform/compatibility-dates for m
|
|
|
632
627
|
id: string | null;
|
|
633
628
|
etag: string | null;
|
|
634
629
|
pipeline_hash: string | null;
|
|
635
|
-
|
|
630
|
+
deployment_id: string | null;
|
|
636
631
|
}>(
|
|
637
632
|
workerUrl,
|
|
638
633
|
{
|
|
@@ -649,7 +644,7 @@ See https://developers.cloudflare.com/workers/platform/compatibility-dates for m
|
|
|
649
644
|
);
|
|
650
645
|
|
|
651
646
|
available_on_subdomain = result.available_on_subdomain;
|
|
652
|
-
|
|
647
|
+
deploymentId = addHyphens(result.deployment_id) ?? result.deployment_id;
|
|
653
648
|
|
|
654
649
|
if (config.first_party_worker) {
|
|
655
650
|
// Print some useful information returned after publishing
|
|
@@ -839,19 +834,7 @@ See https://developers.cloudflare.com/workers/platform/compatibility-dates for m
|
|
|
839
834
|
logger.log("No publish targets for", workerName, formatTime(deployMs));
|
|
840
835
|
}
|
|
841
836
|
|
|
842
|
-
|
|
843
|
-
const deploymentsList = await fetchResult<DeploymentListResult>(
|
|
844
|
-
`/accounts/${accountId}/workers/deployments/by-script/${scriptTag}`
|
|
845
|
-
);
|
|
846
|
-
|
|
847
|
-
logger.log("Current Deployment ID:", deploymentsList.latest.id);
|
|
848
|
-
} catch (e) {
|
|
849
|
-
if ((e as { code: number }).code === 10023) {
|
|
850
|
-
// TODO: remove this try/catch once deployments is completely rolled out
|
|
851
|
-
} else {
|
|
852
|
-
throw e;
|
|
853
|
-
}
|
|
854
|
-
}
|
|
837
|
+
logger.log("Current Deployment ID:", deploymentId);
|
|
855
838
|
}
|
|
856
839
|
|
|
857
840
|
export function helpIfErrorIsSizeOrScriptStartup(
|
|
@@ -1060,6 +1043,7 @@ function updateQueueConsumers(config: Config): Promise<string[]>[] {
|
|
|
1060
1043
|
max_wait_time_ms: consumer.max_batch_timeout
|
|
1061
1044
|
? 1000 * consumer.max_batch_timeout
|
|
1062
1045
|
: undefined,
|
|
1046
|
+
max_concurrency: consumer.max_concurrency,
|
|
1063
1047
|
},
|
|
1064
1048
|
};
|
|
1065
1049
|
|
|
@@ -37,6 +37,11 @@ export function options(yargs: CommonYargsArgv) {
|
|
|
37
37
|
type: "string",
|
|
38
38
|
describe: "Queue to send messages that failed to be consumed",
|
|
39
39
|
},
|
|
40
|
+
"max-concurrency": {
|
|
41
|
+
type: "number",
|
|
42
|
+
describe:
|
|
43
|
+
"The maximum number of concurrent consumer Worker invocations. Must be a positive integer",
|
|
44
|
+
},
|
|
40
45
|
});
|
|
41
46
|
}
|
|
42
47
|
|
|
@@ -55,6 +60,7 @@ export async function handler(
|
|
|
55
60
|
max_wait_time_ms: args.batchTimeout // API expects milliseconds
|
|
56
61
|
? 1000 * args.batchTimeout
|
|
57
62
|
: undefined,
|
|
63
|
+
max_concurrency: args.maxConcurrency,
|
|
58
64
|
},
|
|
59
65
|
dead_letter_queue: args.deadLetterQueue,
|
|
60
66
|
};
|
package/src/queues/client.ts
CHANGED
package/src/secret/index.ts
CHANGED
package/src/sites.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import assert from "node:assert";
|
|
2
2
|
import { readdir, readFile, stat } from "node:fs/promises";
|
|
3
3
|
import * as path from "node:path";
|
|
4
|
+
import chalk from "chalk";
|
|
4
5
|
import ignore from "ignore";
|
|
5
6
|
import xxhash from "xxhash-wasm";
|
|
6
7
|
import {
|
|
@@ -9,8 +10,10 @@ import {
|
|
|
9
10
|
listKVNamespaces,
|
|
10
11
|
putKVBulkKeyValue,
|
|
11
12
|
deleteKVBulkKeyValue,
|
|
13
|
+
BATCH_KEY_MAX,
|
|
14
|
+
formatNumber,
|
|
12
15
|
} from "./kv/helpers";
|
|
13
|
-
import { logger } from "./logger";
|
|
16
|
+
import { logger, LOGGER_LEVELS } from "./logger";
|
|
14
17
|
import type { Config } from "./config";
|
|
15
18
|
import type { KeyValue } from "./kv/helpers";
|
|
16
19
|
import type { XXHashAPI } from "xxhash-wasm";
|
|
@@ -92,6 +95,15 @@ async function createKVNamespaceIfNotAlreadyExisting(
|
|
|
92
95
|
};
|
|
93
96
|
}
|
|
94
97
|
|
|
98
|
+
const MAX_DIFF_LINES = 100;
|
|
99
|
+
const MAX_BUCKET_SIZE = 98 * 1000 * 1000;
|
|
100
|
+
const MAX_BUCKET_KEYS = BATCH_KEY_MAX;
|
|
101
|
+
const MAX_BATCH_OPERATIONS = 5;
|
|
102
|
+
|
|
103
|
+
function pluralise(count: number) {
|
|
104
|
+
return count === 1 ? "" : "s";
|
|
105
|
+
}
|
|
106
|
+
|
|
95
107
|
/**
|
|
96
108
|
* Upload the assets found within the `dirPath` directory to the sites assets KV namespace for
|
|
97
109
|
* the worker given by `scriptName`.
|
|
@@ -116,13 +128,13 @@ export async function syncAssets(
|
|
|
116
128
|
if (siteAssets === undefined) {
|
|
117
129
|
return { manifest: undefined, namespace: undefined };
|
|
118
130
|
}
|
|
119
|
-
|
|
120
131
|
if (dryRun) {
|
|
121
132
|
logger.log("(Note: doing a dry run, not uploading or deleting anything.)");
|
|
122
133
|
return { manifest: undefined, namespace: undefined };
|
|
123
134
|
}
|
|
124
135
|
assert(accountId, "Missing accountId");
|
|
125
136
|
|
|
137
|
+
// Create assets namespace if it doesn't exist
|
|
126
138
|
const title = `__${scriptName}-workers_sites_assets${
|
|
127
139
|
preview ? "_preview" : ""
|
|
128
140
|
}`;
|
|
@@ -131,55 +143,81 @@ export async function syncAssets(
|
|
|
131
143
|
title,
|
|
132
144
|
accountId
|
|
133
145
|
);
|
|
134
|
-
|
|
135
|
-
|
|
146
|
+
// Get all existing keys in asset namespace
|
|
147
|
+
logger.info("Fetching list of already uploaded assets...");
|
|
136
148
|
const namespaceKeysResponse = await listKVNamespaceKeys(accountId, namespace);
|
|
137
149
|
const namespaceKeys = new Set(namespaceKeysResponse.map((x) => x.name));
|
|
138
150
|
|
|
139
|
-
const manifest: Record<string, string> = {};
|
|
140
|
-
|
|
141
|
-
// A batch of uploads where each bucket has to be less than 98mb
|
|
142
|
-
const uploadBuckets: KeyValue[][] = [];
|
|
143
|
-
// The "live" bucket that we'll keep filling until it's just below 98mb
|
|
144
|
-
let uploadBucket: KeyValue[] = [];
|
|
145
|
-
// A size counter for the live bucket
|
|
146
|
-
let uploadBucketSize = 0;
|
|
147
|
-
|
|
148
|
-
const include = createPatternMatcher(siteAssets.includePatterns, false);
|
|
149
|
-
const exclude = createPatternMatcher(siteAssets.excludePatterns, true);
|
|
150
|
-
const hasher = await xxhash();
|
|
151
|
-
|
|
152
151
|
const assetDirectory = path.join(
|
|
153
152
|
siteAssets.baseDirectory,
|
|
154
153
|
siteAssets.assetDirectory
|
|
155
154
|
);
|
|
155
|
+
const include = createPatternMatcher(siteAssets.includePatterns, false);
|
|
156
|
+
const exclude = createPatternMatcher(siteAssets.excludePatterns, true);
|
|
157
|
+
const hasher = await xxhash();
|
|
158
|
+
|
|
159
|
+
// Find and validate all assets before we make any changes (can't store base64
|
|
160
|
+
// contents in memory for upload as users may have *lots* of files, and we
|
|
161
|
+
// don't want to OOM: https://github.com/cloudflare/workers-sdk/issues/2223)
|
|
162
|
+
|
|
163
|
+
const manifest: Record<string, string> = {};
|
|
164
|
+
type PathKey = [path: string, key: string];
|
|
165
|
+
// A batch of uploads where each bucket has to be less than 100 MiB and
|
|
166
|
+
// contain less than 10,000 keys (although we limit to 98 MB and 5000 keys)
|
|
167
|
+
const uploadBuckets: PathKey[][] = [];
|
|
168
|
+
// The "live" bucket we'll keep filling until it's just below the size limit
|
|
169
|
+
let uploadBucket: PathKey[] = [];
|
|
170
|
+
// Current size of the live bucket in bytes (just base64 encoded values)
|
|
171
|
+
let uploadBucketSize = 0;
|
|
172
|
+
|
|
173
|
+
let uploadCount = 0;
|
|
174
|
+
let skipCount = 0;
|
|
175
|
+
|
|
176
|
+
// Always log the first MAX_DIFF_LINES lines, then require the debug log level
|
|
177
|
+
let diffCount = 0;
|
|
178
|
+
function logDiff(line: string) {
|
|
179
|
+
const level = logger.loggerLevel;
|
|
180
|
+
if (LOGGER_LEVELS[level] >= LOGGER_LEVELS.debug) {
|
|
181
|
+
// If we're logging as debug level, we want *all* diff lines to be logged
|
|
182
|
+
// at debug level, not just the first MAX_DIFF_LINES
|
|
183
|
+
logger.debug(line);
|
|
184
|
+
} else if (diffCount < MAX_DIFF_LINES) {
|
|
185
|
+
// Otherwise, log the first MAX_DIFF_LINES diffs at info level...
|
|
186
|
+
logger.info(line);
|
|
187
|
+
} else if (diffCount === MAX_DIFF_LINES) {
|
|
188
|
+
// ...and warn when we start to truncate it
|
|
189
|
+
const msg =
|
|
190
|
+
" (truncating changed assets log, set `WRANGLER_LOG=debug` environment variable to see full diff)";
|
|
191
|
+
logger.info(chalk.dim(msg));
|
|
192
|
+
}
|
|
193
|
+
diffCount++;
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
logger.info("Building list of assets to upload...");
|
|
156
197
|
for await (const absAssetFile of getFilesInFolder(assetDirectory)) {
|
|
157
198
|
const assetFile = path.relative(assetDirectory, absAssetFile);
|
|
158
|
-
if (!include(assetFile))
|
|
159
|
-
continue;
|
|
160
|
-
}
|
|
161
|
-
if (exclude(assetFile)) {
|
|
162
|
-
continue;
|
|
163
|
-
}
|
|
199
|
+
if (!include(assetFile) || exclude(assetFile)) continue;
|
|
164
200
|
|
|
165
|
-
logger.log(`Reading ${assetFile}...`);
|
|
166
201
|
const content = await readFile(absAssetFile, "base64");
|
|
167
|
-
|
|
168
|
-
// while KV accepts files that are 25 MiB **before** b64 encoding
|
|
202
|
+
// While KV accepts files that are 25 MiB **before** b64 encoding
|
|
169
203
|
// the overall bucket size must be below 100 MB **after** b64 encoding
|
|
170
|
-
const assetSize = Buffer.
|
|
204
|
+
const assetSize = Buffer.byteLength(content);
|
|
205
|
+
await validateAssetSize(absAssetFile, assetFile);
|
|
171
206
|
const assetKey = hashAsset(hasher, assetFile, content);
|
|
172
207
|
validateAssetKey(assetKey);
|
|
173
208
|
|
|
174
|
-
// now put each of the files into kv
|
|
175
209
|
if (!namespaceKeys.has(assetKey)) {
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
if
|
|
181
|
-
|
|
182
|
-
|
|
210
|
+
logDiff(
|
|
211
|
+
chalk.green(` + ${assetKey} (uploading new version of ${assetFile})`)
|
|
212
|
+
);
|
|
213
|
+
|
|
214
|
+
// Check if adding this asset to the bucket would push it over the KV
|
|
215
|
+
// bulk API limits
|
|
216
|
+
if (
|
|
217
|
+
uploadBucketSize + assetSize > MAX_BUCKET_SIZE ||
|
|
218
|
+
uploadBucket.length + 1 > MAX_BUCKET_KEYS
|
|
219
|
+
) {
|
|
220
|
+
// If so, record the current bucket and reset it
|
|
183
221
|
uploadBuckets.push(uploadBucket);
|
|
184
222
|
uploadBucketSize = 0;
|
|
185
223
|
uploadBucket = [];
|
|
@@ -187,13 +225,11 @@ export async function syncAssets(
|
|
|
187
225
|
|
|
188
226
|
// Update the bucket and the size counter
|
|
189
227
|
uploadBucketSize += assetSize;
|
|
190
|
-
uploadBucket.push(
|
|
191
|
-
|
|
192
|
-
value: content,
|
|
193
|
-
base64: true,
|
|
194
|
-
});
|
|
228
|
+
uploadBucket.push([absAssetFile, assetKey]);
|
|
229
|
+
uploadCount++;
|
|
195
230
|
} else {
|
|
196
|
-
|
|
231
|
+
logDiff(chalk.dim(` = ${assetKey} (already uploaded ${assetFile})`));
|
|
232
|
+
skipCount++;
|
|
197
233
|
}
|
|
198
234
|
|
|
199
235
|
// Remove the key from the set so we know what we've already uploaded
|
|
@@ -203,23 +239,99 @@ export async function syncAssets(
|
|
|
203
239
|
const manifestKey = urlSafe(path.relative(assetDirectory, absAssetFile));
|
|
204
240
|
manifest[manifestKey] = assetKey;
|
|
205
241
|
}
|
|
242
|
+
// Add the last (potentially only or empty) bucket to the batch
|
|
243
|
+
if (uploadBucket.length > 0) uploadBuckets.push(uploadBucket);
|
|
206
244
|
|
|
207
|
-
// Add the last (potentially only) bucket to the batch
|
|
208
|
-
uploadBuckets.push(uploadBucket);
|
|
209
|
-
|
|
210
|
-
// keys now contains all the files we're deleting
|
|
211
245
|
for (const key of namespaceKeys) {
|
|
212
|
-
|
|
246
|
+
logDiff(chalk.red(` - ${key} (removing as stale)`));
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
// Upload new assets, with 5 concurrent uploaders
|
|
250
|
+
if (uploadCount > 0) {
|
|
251
|
+
const s = pluralise(uploadCount);
|
|
252
|
+
logger.info(`Uploading ${formatNumber(uploadCount)} new asset${s}...`);
|
|
253
|
+
}
|
|
254
|
+
if (skipCount > 0) {
|
|
255
|
+
const s = pluralise(skipCount);
|
|
256
|
+
logger.info(
|
|
257
|
+
`Skipped uploading ${formatNumber(skipCount)} existing asset${s}.`
|
|
258
|
+
);
|
|
213
259
|
}
|
|
260
|
+
let uploadedCount = 0;
|
|
261
|
+
const controller = new AbortController();
|
|
262
|
+
const uploaders = Array.from(Array(MAX_BATCH_OPERATIONS)).map(async () => {
|
|
263
|
+
while (!controller.signal.aborted) {
|
|
264
|
+
// Get the next bucket to upload. If there is none, stop this uploader.
|
|
265
|
+
// JavaScript is single(ish)-threaded, so we don't need to worry about
|
|
266
|
+
// parallel access here.
|
|
267
|
+
const nextBucket = uploadBuckets.shift();
|
|
268
|
+
if (nextBucket === undefined) break;
|
|
269
|
+
|
|
270
|
+
// Read all files in the bucket as base64
|
|
271
|
+
// TODO(perf): consider streaming the bulk upload body, rather than
|
|
272
|
+
// buffering all base64 contents then JSON-stringifying. This probably
|
|
273
|
+
// doesn't matter *too* much: we know buckets will be about 100MB, so
|
|
274
|
+
// with 5 uploaders, we could load about 500MB into memory (+ extra
|
|
275
|
+
// object keys/tags/copies/etc).
|
|
276
|
+
const bucket: KeyValue[] = [];
|
|
277
|
+
for (const [absAssetFile, assetKey] of nextBucket) {
|
|
278
|
+
bucket.push({
|
|
279
|
+
key: assetKey,
|
|
280
|
+
value: await readFile(absAssetFile, "base64"),
|
|
281
|
+
base64: true,
|
|
282
|
+
});
|
|
283
|
+
if (controller.signal.aborted) break;
|
|
284
|
+
}
|
|
214
285
|
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
286
|
+
// Upload the bucket to the KV namespace, suppressing logs, we do our own
|
|
287
|
+
try {
|
|
288
|
+
await putKVBulkKeyValue(
|
|
289
|
+
accountId,
|
|
290
|
+
namespace,
|
|
291
|
+
bucket,
|
|
292
|
+
/* quiet */ true,
|
|
293
|
+
controller.signal
|
|
294
|
+
);
|
|
295
|
+
} catch (e) {
|
|
296
|
+
// https://developer.mozilla.org/en-US/docs/Web/API/DOMException#error_names
|
|
297
|
+
// https://github.com/nodejs/undici/blob/a3efc9814447001a43a976f1c64adc41995df7e3/lib/core/errors.js#L89
|
|
298
|
+
if (
|
|
299
|
+
typeof e === "object" &&
|
|
300
|
+
e !== null &&
|
|
301
|
+
"name" in e &&
|
|
302
|
+
// @ts-expect-error `e.name` should be typed `unknown`, fixed in
|
|
303
|
+
// TypeScript 4.9
|
|
304
|
+
e.name === "AbortError"
|
|
305
|
+
) {
|
|
306
|
+
break;
|
|
307
|
+
}
|
|
308
|
+
throw e;
|
|
309
|
+
}
|
|
310
|
+
uploadedCount += nextBucket.length;
|
|
311
|
+
const percent = Math.floor((100 * uploadedCount) / uploadCount);
|
|
312
|
+
logger.info(
|
|
313
|
+
`Uploaded ${percent}% [${formatNumber(
|
|
314
|
+
uploadedCount
|
|
315
|
+
)} out of ${formatNumber(uploadCount)}]`
|
|
316
|
+
);
|
|
317
|
+
}
|
|
318
|
+
});
|
|
319
|
+
try {
|
|
320
|
+
// Wait for all uploaders to complete, or one to fail
|
|
321
|
+
await Promise.all(uploaders);
|
|
322
|
+
} catch (e) {
|
|
323
|
+
// If any uploader fails, abort the others
|
|
324
|
+
logger.info(`Upload failed, aborting...`);
|
|
325
|
+
controller.abort();
|
|
326
|
+
throw e;
|
|
219
327
|
}
|
|
220
|
-
await Promise.all(bucketsToPut);
|
|
221
328
|
|
|
222
|
-
//
|
|
329
|
+
// Delete stale assets
|
|
330
|
+
const deleteCount = namespaceKeys.size;
|
|
331
|
+
if (deleteCount > 0) {
|
|
332
|
+
const s = pluralise(deleteCount);
|
|
333
|
+
logger.info(`Removing ${formatNumber(deleteCount)} stale asset${s}...`);
|
|
334
|
+
}
|
|
223
335
|
await deleteKVBulkKeyValue(accountId, namespace, Array.from(namespaceKeys));
|
|
224
336
|
|
|
225
337
|
logger.log("↗️ Done syncing assets");
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import { readdir } from "node:fs/promises";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import chalk from "chalk";
|
|
4
|
+
import { logger } from "./logger";
|
|
5
|
+
import { matchFiles, parseRules } from "./module-collection";
|
|
6
|
+
import type { BundleResult } from "./bundle";
|
|
7
|
+
import type { Config } from "./config";
|
|
8
|
+
import type { Entry } from "./entry";
|
|
9
|
+
|
|
10
|
+
async function getFiles(root: string, relativeTo: string): Promise<string[]> {
|
|
11
|
+
const files = [];
|
|
12
|
+
for (const file of await readdir(root, { withFileTypes: true })) {
|
|
13
|
+
if (file.isDirectory()) {
|
|
14
|
+
files.push(...(await getFiles(path.join(root, file.name), relativeTo)));
|
|
15
|
+
} else {
|
|
16
|
+
files.push(path.relative(relativeTo, path.join(root, file.name)));
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
return files;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export default async function traverseModuleGraph(
|
|
23
|
+
entry: Entry,
|
|
24
|
+
rules: Config["rules"]
|
|
25
|
+
): Promise<BundleResult> {
|
|
26
|
+
const files = await getFiles(entry.moduleRoot, entry.moduleRoot);
|
|
27
|
+
const relativeEntryPoint = path.relative(entry.moduleRoot, entry.file);
|
|
28
|
+
|
|
29
|
+
const modules = (await matchFiles(files, entry.moduleRoot, parseRules(rules)))
|
|
30
|
+
.filter((m) => m.name !== relativeEntryPoint)
|
|
31
|
+
.map((m) => ({
|
|
32
|
+
...m,
|
|
33
|
+
name: m.name,
|
|
34
|
+
}));
|
|
35
|
+
|
|
36
|
+
const bundleType = entry.format === "modules" ? "esm" : "commonjs";
|
|
37
|
+
|
|
38
|
+
if (modules.length > 0) {
|
|
39
|
+
logger.info(`Uploading additional modules:`);
|
|
40
|
+
modules.forEach(({ name, type }) => {
|
|
41
|
+
logger.info(`- ${chalk.blue(name)} (${chalk.green(type ?? "")})`);
|
|
42
|
+
});
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
return {
|
|
46
|
+
modules,
|
|
47
|
+
dependencies: {},
|
|
48
|
+
resolvedEntryPointPath: entry.file,
|
|
49
|
+
bundleType,
|
|
50
|
+
stop: undefined,
|
|
51
|
+
sourceMapPath: undefined,
|
|
52
|
+
};
|
|
53
|
+
}
|
package/src/user/user.ts
CHANGED
|
@@ -347,6 +347,7 @@ const Scopes = {
|
|
|
347
347
|
"See and change Cloudflare Pages projects, settings and deployments.",
|
|
348
348
|
"zone:read": "Grants read level access to account zone.",
|
|
349
349
|
"ssl_certs:write": "See and manage mTLS certificates for your account",
|
|
350
|
+
"constellation:write": "Manage Constellation AI projects/models",
|
|
350
351
|
} as const;
|
|
351
352
|
|
|
352
353
|
/**
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
import { EventEmitter } from "events";
|
|
2
|
+
import { render as inkRender } from "ink";
|
|
3
|
+
import type { ReactElement } from "react";
|
|
4
|
+
import type { WriteStream } from "tty";
|
|
5
|
+
|
|
6
|
+
export function renderToString(tree: ReactElement): string {
|
|
7
|
+
const { output, cleanup } = render(tree);
|
|
8
|
+
cleanup();
|
|
9
|
+
return output;
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
// The code below is mostly copied from ink-render-string.
|
|
13
|
+
// See https://github.com/zhanwang626/ink-render-string.git.
|
|
14
|
+
|
|
15
|
+
// MIT License
|
|
16
|
+
|
|
17
|
+
// Copyright (c) 2021 Zhan Wang
|
|
18
|
+
|
|
19
|
+
// Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
20
|
+
// of this software and associated documentation files (the "Software"), to deal
|
|
21
|
+
// in the Software without restriction, including without limitation the rights
|
|
22
|
+
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
23
|
+
// copies of the Software, and to permit persons to whom the Software is
|
|
24
|
+
// furnished to do so, subject to the following conditions:
|
|
25
|
+
|
|
26
|
+
// The above copyright notice and this permission notice shall be included in all
|
|
27
|
+
// copies or substantial portions of the Software.
|
|
28
|
+
|
|
29
|
+
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
30
|
+
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
31
|
+
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
32
|
+
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
33
|
+
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
34
|
+
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
35
|
+
// SOFTWARE.
|
|
36
|
+
|
|
37
|
+
interface Instance {
|
|
38
|
+
output: string;
|
|
39
|
+
unmount: () => void;
|
|
40
|
+
cleanup: () => void;
|
|
41
|
+
stdout: OutputStream;
|
|
42
|
+
stderr: OutputStream;
|
|
43
|
+
frames: string[];
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
const LASTFRAME_UNDEFINED = "value of stdout.lastframe() is undefined";
|
|
47
|
+
|
|
48
|
+
export class OutputStream extends EventEmitter {
|
|
49
|
+
readonly frames: string[] = [];
|
|
50
|
+
private _lastFrame?: string;
|
|
51
|
+
|
|
52
|
+
get columns(): number {
|
|
53
|
+
return this._originalStream.columns;
|
|
54
|
+
}
|
|
55
|
+
get rows(): number {
|
|
56
|
+
return this._originalStream.rows;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
constructor(private _originalStream: WriteStream) {
|
|
60
|
+
super();
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
write = (frame: string) => {
|
|
64
|
+
this.frames.push(frame);
|
|
65
|
+
this._lastFrame = frame;
|
|
66
|
+
};
|
|
67
|
+
|
|
68
|
+
lastFrame = () => {
|
|
69
|
+
return this._lastFrame;
|
|
70
|
+
};
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
export const render = (tree: ReactElement): Instance => {
|
|
74
|
+
const stdout = new OutputStream(process.stdout);
|
|
75
|
+
const stderr = new OutputStream(process.stderr);
|
|
76
|
+
|
|
77
|
+
const instance = inkRender(tree, {
|
|
78
|
+
stdout: stdout as unknown as WriteStream,
|
|
79
|
+
stderr: stderr as unknown as WriteStream,
|
|
80
|
+
debug: true,
|
|
81
|
+
exitOnCtrlC: false,
|
|
82
|
+
patchConsole: false,
|
|
83
|
+
});
|
|
84
|
+
|
|
85
|
+
return {
|
|
86
|
+
output: stdout.lastFrame() || LASTFRAME_UNDEFINED,
|
|
87
|
+
stdout,
|
|
88
|
+
stderr,
|
|
89
|
+
cleanup: instance.cleanup,
|
|
90
|
+
unmount: instance.unmount,
|
|
91
|
+
frames: stdout.frames,
|
|
92
|
+
};
|
|
93
|
+
};
|