wrangler 0.0.0-e33bea9b → 0.0.0-e351afcf

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (105) hide show
  1. package/miniflare-dist/index.mjs +167 -93
  2. package/package.json +14 -9
  3. package/src/__tests__/api-devregistry.test.ts +121 -0
  4. package/src/__tests__/api.test.ts +27 -5
  5. package/src/__tests__/configuration.test.ts +59 -12
  6. package/src/__tests__/constellation.test.ts +371 -0
  7. package/src/__tests__/d1/execute.test.ts +2 -0
  8. package/src/__tests__/d1/migrate.test.ts +102 -0
  9. package/src/__tests__/delete.test.ts +6 -0
  10. package/src/__tests__/deployments.test.ts +335 -95
  11. package/src/__tests__/dev.test.tsx +71 -56
  12. package/src/__tests__/generate.test.ts +3 -4
  13. package/src/__tests__/helpers/mock-console.ts +6 -0
  14. package/src/__tests__/helpers/msw/handlers/deployments.ts +70 -3
  15. package/src/__tests__/helpers/msw/index.ts +4 -2
  16. package/src/__tests__/helpers/worker-scripts/parent-worker.js +4 -1
  17. package/src/__tests__/index.test.ts +14 -6
  18. package/src/__tests__/init.test.ts +127 -96
  19. package/src/__tests__/jest.setup.ts +4 -0
  20. package/src/__tests__/kv.test.ts +9 -9
  21. package/src/__tests__/logger.test.ts +68 -2
  22. package/src/__tests__/middleware.scheduled.test.ts +2 -2
  23. package/src/__tests__/middleware.test.ts +2 -2
  24. package/src/__tests__/mtls-certificates.test.ts +7 -3
  25. package/src/__tests__/pages/deployment-list.test.ts +2 -0
  26. package/src/__tests__/pages/functions-build.test.ts +70 -0
  27. package/src/__tests__/pages/project-list.test.ts +2 -0
  28. package/src/__tests__/pages/project-upload.test.ts +43 -24
  29. package/src/__tests__/pages/publish.test.ts +136 -92
  30. package/src/__tests__/publish.test.ts +878 -332
  31. package/src/__tests__/pubsub.test.ts +3 -0
  32. package/src/__tests__/queues.test.ts +5 -2
  33. package/src/__tests__/traverse-module-graph.test.ts +220 -0
  34. package/src/__tests__/tsconfig.tsbuildinfo +1 -1
  35. package/src/__tests__/user.test.ts +1 -1
  36. package/src/__tests__/worker-namespace.test.ts +1 -0
  37. package/src/api/dev.ts +12 -18
  38. package/src/api/pages/create-worker-bundle-contents.ts +1 -0
  39. package/src/api/pages/publish.tsx +28 -12
  40. package/src/bundle.ts +26 -9
  41. package/src/config/environment.ts +27 -0
  42. package/src/config/index.ts +18 -0
  43. package/src/config/validation.ts +91 -1
  44. package/src/constellation/createProject.tsx +51 -0
  45. package/src/constellation/deleteProject.ts +51 -0
  46. package/src/constellation/deleteProjectModel.ts +68 -0
  47. package/src/constellation/index.ts +75 -0
  48. package/src/constellation/listCatalog.tsx +35 -0
  49. package/src/constellation/listModel.tsx +41 -0
  50. package/src/constellation/listProject.tsx +28 -0
  51. package/src/constellation/listRuntime.tsx +28 -0
  52. package/src/constellation/options.ts +17 -0
  53. package/src/constellation/types.ts +17 -0
  54. package/src/constellation/uploadModel.tsx +64 -0
  55. package/src/constellation/utils.ts +90 -0
  56. package/src/create-worker-upload-form.ts +18 -1
  57. package/src/d1/backups.tsx +19 -15
  58. package/src/d1/create.tsx +45 -39
  59. package/src/d1/delete.ts +1 -1
  60. package/src/d1/execute.tsx +23 -20
  61. package/src/d1/list.tsx +14 -12
  62. package/src/d1/migrations/apply.tsx +36 -30
  63. package/src/d1/migrations/create.tsx +15 -12
  64. package/src/d1/migrations/helpers.ts +5 -8
  65. package/src/d1/migrations/list.tsx +10 -7
  66. package/src/deployments.ts +260 -8
  67. package/src/dev/dev.tsx +2 -0
  68. package/src/dev/local.tsx +48 -20
  69. package/src/dev/start-server.ts +49 -46
  70. package/src/dev/use-esbuild.ts +52 -49
  71. package/src/dev-registry.ts +2 -1
  72. package/src/dev.tsx +11 -5
  73. package/src/docs/helpers.ts +50 -0
  74. package/src/docs/index.ts +25 -68
  75. package/src/entry.ts +19 -8
  76. package/src/environment-variables/factory.ts +2 -1
  77. package/src/generate/index.ts +2 -2
  78. package/src/index.ts +86 -23
  79. package/src/init.ts +144 -135
  80. package/src/kv/helpers.ts +7 -4
  81. package/src/logger.ts +23 -4
  82. package/src/metrics/send-event.ts +2 -1
  83. package/src/miniflare-cli/index.ts +2 -2
  84. package/src/miniflare-cli/tsconfig.tsbuildinfo +1 -1
  85. package/src/module-collection.ts +92 -26
  86. package/src/pages/build.ts +29 -17
  87. package/src/pages/deployments.tsx +3 -5
  88. package/src/pages/dev.ts +24 -6
  89. package/src/pages/functions/buildPlugin.ts +1 -0
  90. package/src/pages/functions/buildWorker.ts +59 -8
  91. package/src/pages/functions/tsconfig.tsbuildinfo +1 -1
  92. package/src/pages/projects.tsx +2 -5
  93. package/src/pages/upload.tsx +24 -4
  94. package/src/publish/publish.ts +10 -26
  95. package/src/queues/cli/commands/consumer/add.ts +6 -0
  96. package/src/queues/client.ts +1 -0
  97. package/src/secret/index.ts +1 -0
  98. package/src/sites.ts +164 -52
  99. package/src/traverse-module-graph.ts +53 -0
  100. package/src/user/user.ts +1 -0
  101. package/src/utils/render.ts +93 -0
  102. package/src/worker.ts +10 -0
  103. package/wrangler-dist/cli.d.ts +29 -2
  104. package/wrangler-dist/cli.js +23270 -22110
  105. package/src/__tests__/api-devregistry.test.js +0 -64
@@ -1,5 +1,4 @@
1
1
  import { execSync } from "node:child_process";
2
- import { render } from "ink";
3
2
  import Table from "ink-table";
4
3
  import React from "react";
5
4
  import { format as timeagoFormat } from "timeago.js";
@@ -10,6 +9,7 @@ import { FatalError } from "../errors";
10
9
  import { logger } from "../logger";
11
10
  import * as metrics from "../metrics";
12
11
  import { requireAuth } from "../user";
12
+ import { renderToString } from "../utils/render";
13
13
  import { PAGES_CONFIG_CACHE_FILENAME } from "./constants";
14
14
  import { pagesBetaWarning } from "./utils";
15
15
  import type {
@@ -44,10 +44,7 @@ export async function ListHandler() {
44
44
  account_id: accountId,
45
45
  });
46
46
 
47
- const { unmount } = render(<Table data={data}></Table>, {
48
- patchConsole: false,
49
- });
50
- unmount();
47
+ logger.log(renderToString(<Table data={data}></Table>));
51
48
  await metrics.sendMetricsEvent("list pages projects");
52
49
  }
53
50
 
@@ -276,9 +276,7 @@ export const upload = async (
276
276
  }
277
277
 
278
278
  let counter = fileMap.size - sortedFiles.length;
279
- const { rerender, unmount } = render(
280
- <Progress done={counter} total={fileMap.size} />
281
- );
279
+ const { rerender, unmount } = renderProgress(counter, fileMap.size);
282
280
 
283
281
  const queue = new PQueue({ concurrency: BULK_UPLOAD_CONCURRENCY });
284
282
 
@@ -336,7 +334,7 @@ export const upload = async (
336
334
  doUpload().then(
337
335
  () => {
338
336
  counter += bucket.files.length;
339
- rerender(<Progress done={counter} total={fileMap.size} />);
337
+ rerender(counter, fileMap.size);
340
338
  },
341
339
  (error) => {
342
340
  return Promise.reject(
@@ -437,6 +435,28 @@ function formatTime(duration: number) {
437
435
  return `(${(duration / 1000).toFixed(2)} sec)`;
438
436
  }
439
437
 
438
+ function renderProgress(done: number, total: number) {
439
+ if (isInteractive()) {
440
+ const { rerender, unmount } = render(
441
+ <Progress done={done} total={total} />
442
+ );
443
+ return {
444
+ // eslint-disable-next-line no-shadow
445
+ rerender(done: number, total: number) {
446
+ rerender(<Progress done={done} total={total} />);
447
+ },
448
+ unmount,
449
+ };
450
+ } else {
451
+ // eslint-disable-next-line no-shadow
452
+ const rerender = (done: number, total: number) => {
453
+ logger.log(`Uploading... (${done}/${total})`);
454
+ };
455
+ rerender(done, total);
456
+ return { rerender, unmount() {} };
457
+ }
458
+ }
459
+
440
460
  function Progress({ done, total }: { done: number; total: number }) {
441
461
  return (
442
462
  <>
@@ -12,6 +12,7 @@ import {
12
12
  import { fetchListResult, fetchResult } from "../cfetch";
13
13
  import { printBindings } from "../config";
14
14
  import { createWorkerUploadForm } from "../create-worker-upload-form";
15
+ import { addHyphens } from "../deployments";
15
16
  import { confirm } from "../dialogs";
16
17
  import { getMigrationsToUpload } from "../durable";
17
18
  import { logger } from "../logger";
@@ -20,6 +21,7 @@ import { ParseError } from "../parse";
20
21
  import { getQueue, putConsumer } from "../queues/client";
21
22
  import { getWorkersDevSubdomain } from "../routes";
22
23
  import { syncAssets } from "../sites";
24
+ import traverseModuleGraph from "../traverse-module-graph";
23
25
  import { identifyD1BindingsAsBeta } from "../worker";
24
26
  import { getZoneForRoute } from "../zones";
25
27
  import type { FetchError } from "../cfetch";
@@ -30,7 +32,6 @@ import type {
30
32
  ZoneNameRoute,
31
33
  CustomDomainRoute,
32
34
  } from "../config/environment";
33
- import type { DeploymentListResult } from "../deployments";
34
35
  import type { Entry } from "../entry";
35
36
  import type { PutConsumerBody } from "../queues/client";
36
37
  import type { AssetPaths } from "../sites";
@@ -335,7 +336,7 @@ See https://developers.cloudflare.com/workers/platform/compatibility-dates for m
335
336
 
336
337
  const jsxFactory = props.jsxFactory || config.jsx_factory;
337
338
  const jsxFragment = props.jsxFragment || config.jsx_fragment;
338
- const keepVars = props.keepVars ?? config.keep_vars;
339
+ const keepVars = props.keepVars || config.keep_vars;
339
340
 
340
341
  const minify = props.minify ?? config.minify;
341
342
 
@@ -401,7 +402,7 @@ See https://developers.cloudflare.com/workers/platform/compatibility-dates for m
401
402
  : `/accounts/${accountId}/workers/scripts/${scriptName}`;
402
403
 
403
404
  let available_on_subdomain: boolean | undefined = undefined; // we'll set this later
404
- let scriptTag: string | null = null;
405
+ let deploymentId: string | null = null;
405
406
 
406
407
  const { format } = props.entry;
407
408
 
@@ -463,14 +464,7 @@ See https://developers.cloudflare.com/workers/platform/compatibility-dates for m
463
464
  resolvedEntryPointPath,
464
465
  bundleType,
465
466
  }: Awaited<ReturnType<typeof bundleWorker>> = props.noBundle
466
- ? // we can skip the whole bundling step and mock a bundle here
467
- {
468
- modules: [],
469
- dependencies: {},
470
- resolvedEntryPointPath: props.entry.file,
471
- bundleType: props.entry.format === "modules" ? "esm" : "commonjs",
472
- stop: undefined,
473
- }
467
+ ? await traverseModuleGraph(props.entry, props.rules)
474
468
  : await bundleWorker(
475
469
  props.entry,
476
470
  typeof destination === "string" ? destination : destination.path,
@@ -541,6 +535,7 @@ See https://developers.cloudflare.com/workers/platform/compatibility-dates for m
541
535
  ? { binding: "__STATIC_CONTENT", id: assets.namespace }
542
536
  : []
543
537
  ),
538
+ send_email: config.send_email,
544
539
  vars: { ...config.vars, ...props.vars },
545
540
  wasm_modules: config.wasm_modules,
546
541
  text_blobs: {
@@ -632,7 +627,7 @@ See https://developers.cloudflare.com/workers/platform/compatibility-dates for m
632
627
  id: string | null;
633
628
  etag: string | null;
634
629
  pipeline_hash: string | null;
635
- tag: string | null;
630
+ deployment_id: string | null;
636
631
  }>(
637
632
  workerUrl,
638
633
  {
@@ -649,7 +644,7 @@ See https://developers.cloudflare.com/workers/platform/compatibility-dates for m
649
644
  );
650
645
 
651
646
  available_on_subdomain = result.available_on_subdomain;
652
- scriptTag = result.tag;
647
+ deploymentId = addHyphens(result.deployment_id) ?? result.deployment_id;
653
648
 
654
649
  if (config.first_party_worker) {
655
650
  // Print some useful information returned after publishing
@@ -839,19 +834,7 @@ See https://developers.cloudflare.com/workers/platform/compatibility-dates for m
839
834
  logger.log("No publish targets for", workerName, formatTime(deployMs));
840
835
  }
841
836
 
842
- try {
843
- const deploymentsList = await fetchResult<DeploymentListResult>(
844
- `/accounts/${accountId}/workers/deployments/by-script/${scriptTag}`
845
- );
846
-
847
- logger.log("Current Deployment ID:", deploymentsList.latest.id);
848
- } catch (e) {
849
- if ((e as { code: number }).code === 10023) {
850
- // TODO: remove this try/catch once deployments is completely rolled out
851
- } else {
852
- throw e;
853
- }
854
- }
837
+ logger.log("Current Deployment ID:", deploymentId);
855
838
  }
856
839
 
857
840
  export function helpIfErrorIsSizeOrScriptStartup(
@@ -1060,6 +1043,7 @@ function updateQueueConsumers(config: Config): Promise<string[]>[] {
1060
1043
  max_wait_time_ms: consumer.max_batch_timeout
1061
1044
  ? 1000 * consumer.max_batch_timeout
1062
1045
  : undefined,
1046
+ max_concurrency: consumer.max_concurrency,
1063
1047
  },
1064
1048
  };
1065
1049
 
@@ -37,6 +37,11 @@ export function options(yargs: CommonYargsArgv) {
37
37
  type: "string",
38
38
  describe: "Queue to send messages that failed to be consumed",
39
39
  },
40
+ "max-concurrency": {
41
+ type: "number",
42
+ describe:
43
+ "The maximum number of concurrent consumer Worker invocations. Must be a positive integer",
44
+ },
40
45
  });
41
46
  }
42
47
 
@@ -55,6 +60,7 @@ export async function handler(
55
60
  max_wait_time_ms: args.batchTimeout // API expects milliseconds
56
61
  ? 1000 * args.batchTimeout
57
62
  : undefined,
63
+ max_concurrency: args.maxConcurrency,
58
64
  },
59
65
  dead_letter_queue: args.deadLetterQueue,
60
66
  };
@@ -90,6 +90,7 @@ export interface ConsumerSettings {
90
90
  batch_size?: number;
91
91
  max_retries?: number;
92
92
  max_wait_time_ms?: number;
93
+ max_concurrency?: number | null;
93
94
  }
94
95
 
95
96
  export interface ConsumerResponse extends PostConsumerBody {
@@ -100,6 +100,7 @@ export const secret = (secretYargs: CommonYargsArgv) => {
100
100
  },
101
101
  bindings: {
102
102
  kv_namespaces: [],
103
+ send_email: [],
103
104
  vars: {},
104
105
  durable_objects: { bindings: [] },
105
106
  queues: [],
package/src/sites.ts CHANGED
@@ -1,6 +1,7 @@
1
1
  import assert from "node:assert";
2
2
  import { readdir, readFile, stat } from "node:fs/promises";
3
3
  import * as path from "node:path";
4
+ import chalk from "chalk";
4
5
  import ignore from "ignore";
5
6
  import xxhash from "xxhash-wasm";
6
7
  import {
@@ -9,8 +10,10 @@ import {
9
10
  listKVNamespaces,
10
11
  putKVBulkKeyValue,
11
12
  deleteKVBulkKeyValue,
13
+ BATCH_KEY_MAX,
14
+ formatNumber,
12
15
  } from "./kv/helpers";
13
- import { logger } from "./logger";
16
+ import { logger, LOGGER_LEVELS } from "./logger";
14
17
  import type { Config } from "./config";
15
18
  import type { KeyValue } from "./kv/helpers";
16
19
  import type { XXHashAPI } from "xxhash-wasm";
@@ -92,6 +95,15 @@ async function createKVNamespaceIfNotAlreadyExisting(
92
95
  };
93
96
  }
94
97
 
98
+ const MAX_DIFF_LINES = 100;
99
+ const MAX_BUCKET_SIZE = 98 * 1000 * 1000;
100
+ const MAX_BUCKET_KEYS = BATCH_KEY_MAX;
101
+ const MAX_BATCH_OPERATIONS = 5;
102
+
103
+ function pluralise(count: number) {
104
+ return count === 1 ? "" : "s";
105
+ }
106
+
95
107
  /**
96
108
  * Upload the assets found within the `dirPath` directory to the sites assets KV namespace for
97
109
  * the worker given by `scriptName`.
@@ -116,13 +128,13 @@ export async function syncAssets(
116
128
  if (siteAssets === undefined) {
117
129
  return { manifest: undefined, namespace: undefined };
118
130
  }
119
-
120
131
  if (dryRun) {
121
132
  logger.log("(Note: doing a dry run, not uploading or deleting anything.)");
122
133
  return { manifest: undefined, namespace: undefined };
123
134
  }
124
135
  assert(accountId, "Missing accountId");
125
136
 
137
+ // Create assets namespace if it doesn't exist
126
138
  const title = `__${scriptName}-workers_sites_assets${
127
139
  preview ? "_preview" : ""
128
140
  }`;
@@ -131,55 +143,81 @@ export async function syncAssets(
131
143
  title,
132
144
  accountId
133
145
  );
134
-
135
- // let's get all the keys in this namespace
146
+ // Get all existing keys in asset namespace
147
+ logger.info("Fetching list of already uploaded assets...");
136
148
  const namespaceKeysResponse = await listKVNamespaceKeys(accountId, namespace);
137
149
  const namespaceKeys = new Set(namespaceKeysResponse.map((x) => x.name));
138
150
 
139
- const manifest: Record<string, string> = {};
140
-
141
- // A batch of uploads where each bucket has to be less than 98mb
142
- const uploadBuckets: KeyValue[][] = [];
143
- // The "live" bucket that we'll keep filling until it's just below 98mb
144
- let uploadBucket: KeyValue[] = [];
145
- // A size counter for the live bucket
146
- let uploadBucketSize = 0;
147
-
148
- const include = createPatternMatcher(siteAssets.includePatterns, false);
149
- const exclude = createPatternMatcher(siteAssets.excludePatterns, true);
150
- const hasher = await xxhash();
151
-
152
151
  const assetDirectory = path.join(
153
152
  siteAssets.baseDirectory,
154
153
  siteAssets.assetDirectory
155
154
  );
155
+ const include = createPatternMatcher(siteAssets.includePatterns, false);
156
+ const exclude = createPatternMatcher(siteAssets.excludePatterns, true);
157
+ const hasher = await xxhash();
158
+
159
+ // Find and validate all assets before we make any changes (can't store base64
160
+ // contents in memory for upload as users may have *lots* of files, and we
161
+ // don't want to OOM: https://github.com/cloudflare/workers-sdk/issues/2223)
162
+
163
+ const manifest: Record<string, string> = {};
164
+ type PathKey = [path: string, key: string];
165
+ // A batch of uploads where each bucket has to be less than 100 MiB and
166
+ // contain less than 10,000 keys (although we limit to 98 MB and 5000 keys)
167
+ const uploadBuckets: PathKey[][] = [];
168
+ // The "live" bucket we'll keep filling until it's just below the size limit
169
+ let uploadBucket: PathKey[] = [];
170
+ // Current size of the live bucket in bytes (just base64 encoded values)
171
+ let uploadBucketSize = 0;
172
+
173
+ let uploadCount = 0;
174
+ let skipCount = 0;
175
+
176
+ // Always log the first MAX_DIFF_LINES lines, then require the debug log level
177
+ let diffCount = 0;
178
+ function logDiff(line: string) {
179
+ const level = logger.loggerLevel;
180
+ if (LOGGER_LEVELS[level] >= LOGGER_LEVELS.debug) {
181
+ // If we're logging as debug level, we want *all* diff lines to be logged
182
+ // at debug level, not just the first MAX_DIFF_LINES
183
+ logger.debug(line);
184
+ } else if (diffCount < MAX_DIFF_LINES) {
185
+ // Otherwise, log the first MAX_DIFF_LINES diffs at info level...
186
+ logger.info(line);
187
+ } else if (diffCount === MAX_DIFF_LINES) {
188
+ // ...and warn when we start to truncate it
189
+ const msg =
190
+ " (truncating changed assets log, set `WRANGLER_LOG=debug` environment variable to see full diff)";
191
+ logger.info(chalk.dim(msg));
192
+ }
193
+ diffCount++;
194
+ }
195
+
196
+ logger.info("Building list of assets to upload...");
156
197
  for await (const absAssetFile of getFilesInFolder(assetDirectory)) {
157
198
  const assetFile = path.relative(assetDirectory, absAssetFile);
158
- if (!include(assetFile)) {
159
- continue;
160
- }
161
- if (exclude(assetFile)) {
162
- continue;
163
- }
199
+ if (!include(assetFile) || exclude(assetFile)) continue;
164
200
 
165
- logger.log(`Reading ${assetFile}...`);
166
201
  const content = await readFile(absAssetFile, "base64");
167
- await validateAssetSize(absAssetFile, assetFile);
168
- // while KV accepts files that are 25 MiB **before** b64 encoding
202
+ // While KV accepts files that are 25 MiB **before** b64 encoding
169
203
  // the overall bucket size must be below 100 MB **after** b64 encoding
170
- const assetSize = Buffer.from(content).length;
204
+ const assetSize = Buffer.byteLength(content);
205
+ await validateAssetSize(absAssetFile, assetFile);
171
206
  const assetKey = hashAsset(hasher, assetFile, content);
172
207
  validateAssetKey(assetKey);
173
208
 
174
- // now put each of the files into kv
175
209
  if (!namespaceKeys.has(assetKey)) {
176
- logger.log(`Uploading as ${assetKey}...`);
177
-
178
- // Check if adding this asset to the bucket would
179
- // push it over the 98 MiB limit KV bulk API limit
180
- if (uploadBucketSize + assetSize > 98 * 1000 * 1000) {
181
- // If so, move the current bucket into the batch,
182
- // and reset the counter/bucket
210
+ logDiff(
211
+ chalk.green(` + ${assetKey} (uploading new version of ${assetFile})`)
212
+ );
213
+
214
+ // Check if adding this asset to the bucket would push it over the KV
215
+ // bulk API limits
216
+ if (
217
+ uploadBucketSize + assetSize > MAX_BUCKET_SIZE ||
218
+ uploadBucket.length + 1 > MAX_BUCKET_KEYS
219
+ ) {
220
+ // If so, record the current bucket and reset it
183
221
  uploadBuckets.push(uploadBucket);
184
222
  uploadBucketSize = 0;
185
223
  uploadBucket = [];
@@ -187,13 +225,11 @@ export async function syncAssets(
187
225
 
188
226
  // Update the bucket and the size counter
189
227
  uploadBucketSize += assetSize;
190
- uploadBucket.push({
191
- key: assetKey,
192
- value: content,
193
- base64: true,
194
- });
228
+ uploadBucket.push([absAssetFile, assetKey]);
229
+ uploadCount++;
195
230
  } else {
196
- logger.log(`Skipping - already uploaded.`);
231
+ logDiff(chalk.dim(` = ${assetKey} (already uploaded ${assetFile})`));
232
+ skipCount++;
197
233
  }
198
234
 
199
235
  // Remove the key from the set so we know what we've already uploaded
@@ -203,23 +239,99 @@ export async function syncAssets(
203
239
  const manifestKey = urlSafe(path.relative(assetDirectory, absAssetFile));
204
240
  manifest[manifestKey] = assetKey;
205
241
  }
242
+ // Add the last (potentially only or empty) bucket to the batch
243
+ if (uploadBucket.length > 0) uploadBuckets.push(uploadBucket);
206
244
 
207
- // Add the last (potentially only) bucket to the batch
208
- uploadBuckets.push(uploadBucket);
209
-
210
- // keys now contains all the files we're deleting
211
245
  for (const key of namespaceKeys) {
212
- logger.log(`Deleting ${key} from the asset store...`);
246
+ logDiff(chalk.red(` - ${key} (removing as stale)`));
247
+ }
248
+
249
+ // Upload new assets, with 5 concurrent uploaders
250
+ if (uploadCount > 0) {
251
+ const s = pluralise(uploadCount);
252
+ logger.info(`Uploading ${formatNumber(uploadCount)} new asset${s}...`);
253
+ }
254
+ if (skipCount > 0) {
255
+ const s = pluralise(skipCount);
256
+ logger.info(
257
+ `Skipped uploading ${formatNumber(skipCount)} existing asset${s}.`
258
+ );
213
259
  }
260
+ let uploadedCount = 0;
261
+ const controller = new AbortController();
262
+ const uploaders = Array.from(Array(MAX_BATCH_OPERATIONS)).map(async () => {
263
+ while (!controller.signal.aborted) {
264
+ // Get the next bucket to upload. If there is none, stop this uploader.
265
+ // JavaScript is single(ish)-threaded, so we don't need to worry about
266
+ // parallel access here.
267
+ const nextBucket = uploadBuckets.shift();
268
+ if (nextBucket === undefined) break;
269
+
270
+ // Read all files in the bucket as base64
271
+ // TODO(perf): consider streaming the bulk upload body, rather than
272
+ // buffering all base64 contents then JSON-stringifying. This probably
273
+ // doesn't matter *too* much: we know buckets will be about 100MB, so
274
+ // with 5 uploaders, we could load about 500MB into memory (+ extra
275
+ // object keys/tags/copies/etc).
276
+ const bucket: KeyValue[] = [];
277
+ for (const [absAssetFile, assetKey] of nextBucket) {
278
+ bucket.push({
279
+ key: assetKey,
280
+ value: await readFile(absAssetFile, "base64"),
281
+ base64: true,
282
+ });
283
+ if (controller.signal.aborted) break;
284
+ }
214
285
 
215
- // upload each bucket in parallel
216
- const bucketsToPut = [];
217
- for (const bucket of uploadBuckets) {
218
- bucketsToPut.push(putKVBulkKeyValue(accountId, namespace, bucket));
286
+ // Upload the bucket to the KV namespace, suppressing logs, we do our own
287
+ try {
288
+ await putKVBulkKeyValue(
289
+ accountId,
290
+ namespace,
291
+ bucket,
292
+ /* quiet */ true,
293
+ controller.signal
294
+ );
295
+ } catch (e) {
296
+ // https://developer.mozilla.org/en-US/docs/Web/API/DOMException#error_names
297
+ // https://github.com/nodejs/undici/blob/a3efc9814447001a43a976f1c64adc41995df7e3/lib/core/errors.js#L89
298
+ if (
299
+ typeof e === "object" &&
300
+ e !== null &&
301
+ "name" in e &&
302
+ // @ts-expect-error `e.name` should be typed `unknown`, fixed in
303
+ // TypeScript 4.9
304
+ e.name === "AbortError"
305
+ ) {
306
+ break;
307
+ }
308
+ throw e;
309
+ }
310
+ uploadedCount += nextBucket.length;
311
+ const percent = Math.floor((100 * uploadedCount) / uploadCount);
312
+ logger.info(
313
+ `Uploaded ${percent}% [${formatNumber(
314
+ uploadedCount
315
+ )} out of ${formatNumber(uploadCount)}]`
316
+ );
317
+ }
318
+ });
319
+ try {
320
+ // Wait for all uploaders to complete, or one to fail
321
+ await Promise.all(uploaders);
322
+ } catch (e) {
323
+ // If any uploader fails, abort the others
324
+ logger.info(`Upload failed, aborting...`);
325
+ controller.abort();
326
+ throw e;
219
327
  }
220
- await Promise.all(bucketsToPut);
221
328
 
222
- // then delete all the assets that aren't used anymore
329
+ // Delete stale assets
330
+ const deleteCount = namespaceKeys.size;
331
+ if (deleteCount > 0) {
332
+ const s = pluralise(deleteCount);
333
+ logger.info(`Removing ${formatNumber(deleteCount)} stale asset${s}...`);
334
+ }
223
335
  await deleteKVBulkKeyValue(accountId, namespace, Array.from(namespaceKeys));
224
336
 
225
337
  logger.log("↗️ Done syncing assets");
@@ -0,0 +1,53 @@
1
+ import { readdir } from "node:fs/promises";
2
+ import path from "node:path";
3
+ import chalk from "chalk";
4
+ import { logger } from "./logger";
5
+ import { matchFiles, parseRules } from "./module-collection";
6
+ import type { BundleResult } from "./bundle";
7
+ import type { Config } from "./config";
8
+ import type { Entry } from "./entry";
9
+
10
+ async function getFiles(root: string, relativeTo: string): Promise<string[]> {
11
+ const files = [];
12
+ for (const file of await readdir(root, { withFileTypes: true })) {
13
+ if (file.isDirectory()) {
14
+ files.push(...(await getFiles(path.join(root, file.name), relativeTo)));
15
+ } else {
16
+ files.push(path.relative(relativeTo, path.join(root, file.name)));
17
+ }
18
+ }
19
+ return files;
20
+ }
21
+
22
+ export default async function traverseModuleGraph(
23
+ entry: Entry,
24
+ rules: Config["rules"]
25
+ ): Promise<BundleResult> {
26
+ const files = await getFiles(entry.moduleRoot, entry.moduleRoot);
27
+ const relativeEntryPoint = path.relative(entry.moduleRoot, entry.file);
28
+
29
+ const modules = (await matchFiles(files, entry.moduleRoot, parseRules(rules)))
30
+ .filter((m) => m.name !== relativeEntryPoint)
31
+ .map((m) => ({
32
+ ...m,
33
+ name: m.name,
34
+ }));
35
+
36
+ const bundleType = entry.format === "modules" ? "esm" : "commonjs";
37
+
38
+ if (modules.length > 0) {
39
+ logger.info(`Uploading additional modules:`);
40
+ modules.forEach(({ name, type }) => {
41
+ logger.info(`- ${chalk.blue(name)} (${chalk.green(type ?? "")})`);
42
+ });
43
+ }
44
+
45
+ return {
46
+ modules,
47
+ dependencies: {},
48
+ resolvedEntryPointPath: entry.file,
49
+ bundleType,
50
+ stop: undefined,
51
+ sourceMapPath: undefined,
52
+ };
53
+ }
package/src/user/user.ts CHANGED
@@ -347,6 +347,7 @@ const Scopes = {
347
347
  "See and change Cloudflare Pages projects, settings and deployments.",
348
348
  "zone:read": "Grants read level access to account zone.",
349
349
  "ssl_certs:write": "See and manage mTLS certificates for your account",
350
+ "constellation:write": "Manage Constellation AI projects/models",
350
351
  } as const;
351
352
 
352
353
  /**
@@ -0,0 +1,93 @@
1
+ import { EventEmitter } from "events";
2
+ import { render as inkRender } from "ink";
3
+ import type { ReactElement } from "react";
4
+ import type { WriteStream } from "tty";
5
+
6
+ export function renderToString(tree: ReactElement): string {
7
+ const { output, cleanup } = render(tree);
8
+ cleanup();
9
+ return output;
10
+ }
11
+
12
+ // The code below is mostly copied from ink-render-string.
13
+ // See https://github.com/zhanwang626/ink-render-string.git.
14
+
15
+ // MIT License
16
+
17
+ // Copyright (c) 2021 Zhan Wang
18
+
19
+ // Permission is hereby granted, free of charge, to any person obtaining a copy
20
+ // of this software and associated documentation files (the "Software"), to deal
21
+ // in the Software without restriction, including without limitation the rights
22
+ // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
23
+ // copies of the Software, and to permit persons to whom the Software is
24
+ // furnished to do so, subject to the following conditions:
25
+
26
+ // The above copyright notice and this permission notice shall be included in all
27
+ // copies or substantial portions of the Software.
28
+
29
+ // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
30
+ // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
31
+ // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
32
+ // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
33
+ // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
34
+ // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
35
+ // SOFTWARE.
36
+
37
+ interface Instance {
38
+ output: string;
39
+ unmount: () => void;
40
+ cleanup: () => void;
41
+ stdout: OutputStream;
42
+ stderr: OutputStream;
43
+ frames: string[];
44
+ }
45
+
46
+ const LASTFRAME_UNDEFINED = "value of stdout.lastframe() is undefined";
47
+
48
+ export class OutputStream extends EventEmitter {
49
+ readonly frames: string[] = [];
50
+ private _lastFrame?: string;
51
+
52
+ get columns(): number {
53
+ return this._originalStream.columns;
54
+ }
55
+ get rows(): number {
56
+ return this._originalStream.rows;
57
+ }
58
+
59
+ constructor(private _originalStream: WriteStream) {
60
+ super();
61
+ }
62
+
63
+ write = (frame: string) => {
64
+ this.frames.push(frame);
65
+ this._lastFrame = frame;
66
+ };
67
+
68
+ lastFrame = () => {
69
+ return this._lastFrame;
70
+ };
71
+ }
72
+
73
+ export const render = (tree: ReactElement): Instance => {
74
+ const stdout = new OutputStream(process.stdout);
75
+ const stderr = new OutputStream(process.stderr);
76
+
77
+ const instance = inkRender(tree, {
78
+ stdout: stdout as unknown as WriteStream,
79
+ stderr: stderr as unknown as WriteStream,
80
+ debug: true,
81
+ exitOnCtrlC: false,
82
+ patchConsole: false,
83
+ });
84
+
85
+ return {
86
+ output: stdout.lastFrame() || LASTFRAME_UNDEFINED,
87
+ stdout,
88
+ stderr,
89
+ cleanup: instance.cleanup,
90
+ unmount: instance.unmount,
91
+ frames: stdout.frames,
92
+ };
93
+ };