rollbridge 0.1.70 → 0.1.71

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. package/README.md +10 -63
  2. package/TODO.md +0 -13
  3. package/docs/cli.md +3 -128
  4. package/docs/config.md +6 -177
  5. package/docs/troubleshooting.md +6 -97
  6. package/docs/velocious.md +1 -1
  7. package/docs/workers.md +0 -11
  8. package/examples/tensorbuzz.com.js +2 -3
  9. package/package.json +1 -1
  10. package/src/cli.js +2 -68
  11. package/src/config.js +8 -347
  12. package/src/daemon.js +65 -387
  13. package/src/doctor.js +0 -74
  14. package/src/guardian-client.js +8 -67
  15. package/src/health.js +8 -99
  16. package/src/managed-process.js +16 -197
  17. package/src/process-guardian.js +22 -212
  18. package/src/process-memory.js +2 -62
  19. package/src/release-group.js +37 -435
  20. package/tensorbuzz.yml +1 -20
  21. package/test/completion.test.js +2 -5
  22. package/test/config-validation.test.js +5 -135
  23. package/test/guardian-client.test.js +5 -119
  24. package/test/health.test.js +1 -74
  25. package/test/managed-process.test.js +1 -260
  26. package/test/owner-recovery.test.js +21 -125
  27. package/test/owner-replacement.test.js +14 -175
  28. package/test/process-memory.test.js +1 -47
  29. package/test/release-group.test.js +2 -408
  30. package/test/rollbridge.test.js +72 -737
  31. package/test/shutdown-completion.test.js +0 -52
  32. package/test/support/process.js +1 -3
  33. package/changelog.d/20261025-guardian-re-register-dead-generation.md +0 -13
  34. package/changelog.d/20261026-stale-committed-generation-transition-recovery.md +0 -1
  35. package/changelog.d/20261104-compose-service-units.md +0 -21
  36. package/changelog.d/20261105-owner-mutation-lease-diagnostics.md +0 -20
  37. package/changelog.d/20261106-release-dependency-boot-budgets.md +0 -32
  38. package/changelog.d/20261109-path-filtered-builds.md +0 -11
  39. package/changelog.d/20261110-default-boot-budget-above-measured-startup.md +0 -26
  40. package/changelog.d/20261111-contention-sensitive-daemon-specs.md +0 -27
  41. package/changelog.d/20261113-bounded-nonblocking-retirement.md +0 -5
  42. package/changelog.d/20261115-bound-draining-release-groups.md +0 -62
  43. package/changelog.d/20261115-restore-deploy-refuses-held-generation.md +0 -27
  44. package/changelog.d/20261115-wedged-generation-escalation.md +0 -44
  45. package/changelog.d/20261116-bounded-explicit-stop-retirement.md +0 -30
  46. package/changelog.d/20261116-config-json-canonical-shape.md +0 -11
  47. package/changelog.d/20261116-drain-specs-gate-the-retirement-record.md +0 -29
  48. package/changelog.d/20261116-guardian-version-handoff.md +0 -37
  49. package/changelog.d/20261116-reclaim-dead-owner-mutation.md +0 -7
  50. package/changelog.d/20261117-activate-before-incumbent-retirement.md +0 -1
  51. package/changelog.d/20261117-activation-path-drain-deadline.md +0 -35
  52. package/changelog.d/20261117-stop-companion-after-retired-dependency.md +0 -7
  53. package/docs/compose-services.md +0 -222
  54. package/src/compose-unit.js +0 -725
  55. package/src/drain-policy.js +0 -252
  56. package/src/process-order.js +0 -26
  57. package/test/compose-config.test.js +0 -160
  58. package/test/compose-docker.test.js +0 -297
  59. package/test/compose-unit.test.js +0 -415
  60. package/test/drain-policy.test.js +0 -328
  61. package/test/fixtures/deferred-boot-app.js +0 -106
  62. package/test/fixtures/gated-app.js +0 -50
  63. package/test/tensorbuzz-config.test.js +0 -228
@@ -1,415 +0,0 @@
1
- // @ts-check
2
-
3
- import fs from "node:fs/promises"
4
- import os from "node:os"
5
- import path from "node:path"
6
- import {describe, expect, test} from "@velocious/testing"
7
- import {normalizeConfig} from "../src/config.js"
8
- import {composeUnitStatus, readComposeLedger, rollbackComposeUnit, rollComposeUnit} from "../src/compose-unit.js"
9
-
10
- describe("compose-unit", () => {
11
-
12
- /**
13
- * @typedef {{code: number, signal: string | null, stderr: string, stdout: string}} CommandResult
14
- * @typedef {{args?: string[], command?: string}} RecordedCall
15
- * @typedef {{containerId: string, health: string, image: string | null, ports: {HostPort: string, containerPort: string}[], recreateImages: string[], state: string}} DockerState
16
- * @typedef {{healthFor?: (image: string) => string, image: string, ports?: {HostPort: string, containerPort: string}[], recreateError?: string, shell?: (command: string) => CommandResult}} DockerWorldOptions
17
- * @typedef {{calls: RecordedCall[], composeFile: string, projectDirectory: string, runCommand: import("../src/compose-unit.js").CommandRunner, state: DockerState}} DockerWorld
18
- * @typedef {import("../src/config.js").RollbridgeConfig & {compose: import("../src/config.js").ComposeUnitConfig[]}} ComposeTestConfig
19
- */
20
-
21
- const CURRENT_IMAGE = "registry.local/gateway:1.4.0"
22
- const TARGET_IMAGE = "registry.local/gateway:2.0.0"
23
- const OLD_IMAGE = "registry.local/gateway:1.0.0"
24
-
25
- /**
26
- * Stand-in for the docker/docker-compose CLI: it records every invocation and mutates a small
27
- * container model, so a roll can be proven stage by stage without a Docker daemon.
28
- * @param {DockerWorldOptions} options - Initial container state and per-command behaviour.
29
- * @returns {Promise<DockerWorld>} World with its recorded calls and bound command runner.
30
- */
31
- async function createDockerWorld(options) {
32
- const projectDirectory = await fs.mkdtemp(path.join(os.tmpdir(), "rollbridge-compose-unit-"))
33
- const composeFile = path.join(projectDirectory, "gateway.compose.yml")
34
-
35
- await fs.writeFile(composeFile, `services:\n gateway:\n image: ${CURRENT_IMAGE}\n`)
36
-
37
- /** @type {DockerState} */
38
- const state = {
39
- containerId: "container-1",
40
- health: "healthy",
41
- image: options.image,
42
- ports: options.ports ?? [],
43
- recreateImages: [],
44
- state: "running"
45
- }
46
- /** @type {RecordedCall[]} */
47
- const calls = []
48
-
49
- /** @type {import("../src/compose-unit.js").CommandRunner} */
50
- const runCommand = async ({args, command}) => {
51
- if (command !== undefined) {
52
- calls.push({command})
53
-
54
- return options.shell?.(command) ?? {code: 0, signal: null, stderr: "", stdout: ""}
55
- }
56
-
57
- calls.push({args})
58
-
59
- const [executable, ...rest] = args ?? []
60
-
61
- if (executable === "docker" && rest[0] === "compose") {
62
- if (rest.includes("ps")) return {code: 0, signal: null, stderr: "", stdout: `${state.containerId}\n`}
63
-
64
- if (rest.includes("up")) {
65
- const overridePath = rest[rest.findIndex((token) => token.endsWith(".override.yml"))]
66
- const override = await fs.readFile(overridePath, "utf8")
67
- const image = /** @type {string[]} */ (override.match(/image: "([^"]+)"/))[1]
68
-
69
- // A compose recreate that the daemon refuses does not touch the running container, which is
70
- // what the real-daemon proof measures; the stand-in keeps the same observable state.
71
- if (options.recreateError !== undefined) {
72
- return {code: 1, signal: null, stderr: `${options.recreateError} (image ${image})`, stdout: ""}
73
- }
74
-
75
- state.recreateImages.push(image)
76
- state.image = image
77
- state.containerId = `container-${state.recreateImages.length + 1}`
78
- state.health = options.healthFor?.(image) ?? "healthy"
79
-
80
- return {code: 0, signal: null, stderr: "", stdout: ""}
81
- }
82
- }
83
-
84
- if (executable === "docker" && rest[0] === "inspect") {
85
- const portJson = state.ports.length === 0
86
- ? "null"
87
- : JSON.stringify(Object.fromEntries(state.ports.map((port) => [`${port.containerPort}/tcp`, [{HostIp: "0.0.0.0", HostPort: port.HostPort}]])))
88
-
89
- return {code: 0, signal: null, stderr: "", stdout: `${state.image}|${state.state}|${state.health}|${portJson}|null\n`}
90
- }
91
-
92
- return {code: 127, signal: null, stderr: `unexpected command: ${(args ?? []).join(" ")}`, stdout: ""}
93
- }
94
-
95
- return {calls, composeFile, projectDirectory, runCommand, state}
96
- }
97
-
98
- /**
99
- * @param {string} projectDirectory - Directory holding the Compose file.
100
- * @param {Record<string, import("../src/json.js").JsonValue>} [composeOverrides] - Unit field overrides.
101
- * @returns {ComposeTestConfig} Config with one Compose unit.
102
- */
103
- function buildConfig(projectDirectory, composeOverrides = {}) {
104
- return /** @type {ComposeTestConfig} */ (normalizeConfig({
105
- application: "compose-test",
106
- compose: [{
107
- activation: {intervalMs: 20, timeoutMs: 200},
108
- drain: {command: "quiesce {{service}}", timeoutMs: 500},
109
- id: "gateway",
110
- image: TARGET_IMAGE,
111
- inFlight: {command: "await in-flight {{service}}", intervalMs: 20, timeoutMs: 200},
112
- projectDirectory,
113
- service: "gateway",
114
- stopGracePeriodMs: 30000,
115
- ...composeOverrides
116
- }],
117
- composeStatePath: path.join(projectDirectory, "compose.state.json"),
118
- control: {path: path.join(projectDirectory, "rollbridge.sock")},
119
- processes: [{command: "true", id: "web", policy: "proxied", port: {from: 0, to: 0}}],
120
- proxy: {host: "127.0.0.1", port: 8182}
121
- }))
122
- }
123
-
124
- /**
125
- * @param {string} prefix - Command prefix that should fail.
126
- * @param {string} stderr - Failure output.
127
- * @returns {(command: string) => CommandResult} Shell handler failing every command with this prefix.
128
- */
129
- function failingShell(prefix, stderr) {
130
- return (command) => command.startsWith(prefix)
131
- ? {code: 1, signal: null, stderr, stdout: ""}
132
- : {code: 0, signal: null, stderr: "", stdout: ""}
133
- }
134
-
135
- test("rolls one Compose unit drain-first and retains the previous image", async () => {
136
- const world = await createDockerWorld({image: CURRENT_IMAGE})
137
- const config = buildConfig(world.projectDirectory)
138
- const unit = config.compose[0]
139
-
140
- try {
141
- const result = await rollComposeUnit({config, runCommand: world.runCommand, unit})
142
-
143
- expect(result.status).toBe("rolled")
144
- expect(result.stage).toBe("activation")
145
- expect(result.image).toBe(TARGET_IMAGE)
146
- expect(result.previousImage).toBe(CURRENT_IMAGE)
147
- expect(result.generations.map((generation) => [generation.image, generation.status])).toEqual([[TARGET_IMAGE, "activated"]])
148
- expect(world.state.recreateImages).toEqual([TARGET_IMAGE])
149
-
150
- const commands = world.calls.map((call) => call.command).filter(Boolean)
151
-
152
- expect(commands).toEqual(["quiesce gateway", "await in-flight gateway"])
153
-
154
- const recreateArgs = /** @type {string[]} */ (world.calls.find((call) => call.args?.includes("up"))?.args)
155
-
156
- expect(recreateArgs.slice(-5)).toEqual(["up", "-d", "--no-deps", "--force-recreate", "gateway"])
157
- expect(recreateArgs).toContain("--no-deps")
158
-
159
- const overridePath = recreateArgs[recreateArgs.findIndex((token) => token.endsWith(".override.yml"))]
160
-
161
- expect(overridePath.startsWith(world.projectDirectory)).toBeTruthy()
162
-
163
- const ledger = await readComposeLedger({config, unit})
164
-
165
- expect(ledger.generations[0].previousImage).toBe(CURRENT_IMAGE)
166
- expect(ledger.generations[0].releaseId).toBe(result.releaseId)
167
- await expect(fs.stat(overridePath)).rejects.toThrow()
168
- } finally {
169
- await fs.rm(world.projectDirectory, {force: true, recursive: true})
170
- }
171
- })
172
-
173
- test("the generated override pins the target image and the stop grace period", async () => {
174
- const world = await createDockerWorld({image: CURRENT_IMAGE})
175
- const config = buildConfig(world.projectDirectory)
176
- /** @type {string | undefined} */
177
- let overrideContents
178
-
179
- try {
180
- /** @type {import("../src/compose-unit.js").CommandRunner} */
181
- const recordingRunner = async (request) => {
182
- if (request.args?.includes("up")) {
183
- overrideContents = await fs.readFile(request.args[request.args.findIndex((token) => token.endsWith(".override.yml"))], "utf8")
184
- }
185
-
186
- return await world.runCommand(request)
187
- }
188
-
189
- const result = await rollComposeUnit({config, runCommand: recordingRunner, unit: config.compose[0]})
190
-
191
- expect(result.status).toBe("rolled")
192
- expect(overrideContents).toMatch(/image: "registry\.local\/gateway:2\.0\.0"/)
193
- expect(overrideContents).toMatch(/stop_grace_period: 30s/)
194
- } finally {
195
- await fs.rm(world.projectDirectory, {force: true, recursive: true})
196
- }
197
- })
198
-
199
- test("a unit that cannot be quiesced fails the roll and is never recreated", async () => {
200
- const world = await createDockerWorld({image: CURRENT_IMAGE, shell: failingShell("quiesce", "gateway refused to drain: drain socket timeout")})
201
- const config = buildConfig(world.projectDirectory)
202
- const unit = config.compose[0]
203
-
204
- try {
205
- const result = await rollComposeUnit({config, runCommand: world.runCommand, unit})
206
-
207
- expect(result.status).toBe("failed")
208
- expect(result.stage).toBe("drain")
209
- expect(result.error).toMatch(/could not be quiesced/)
210
- expect(result.error).toMatch(/gateway refused to drain/)
211
- expect(world.state.recreateImages).toEqual([])
212
- expect(world.state.image).toBe(CURRENT_IMAGE)
213
- expect(world.state.containerId).toBe("container-1")
214
- expect(world.calls.some((call) => call.args?.includes("up"))).toBe(false)
215
- expect((await readComposeLedger({config, unit})).generations).toEqual([])
216
- } finally {
217
- await fs.rm(world.projectDirectory, {force: true, recursive: true})
218
- }
219
- })
220
-
221
- test("in-flight work that never settles fails the roll and keeps the current container", async () => {
222
- const world = await createDockerWorld({image: CURRENT_IMAGE, shell: failingShell("await", "3 handoffs still running")})
223
- const config = buildConfig(world.projectDirectory, {inFlight: {command: "await in-flight {{service}}", intervalMs: 20, timeoutMs: 60}})
224
-
225
- try {
226
- const result = await rollComposeUnit({config, runCommand: world.runCommand, unit: config.compose[0]})
227
-
228
- expect(result.status).toBe("failed")
229
- expect(result.stage).toBe("in_flight")
230
- expect(result.error).toMatch(/still had in-flight work after 60ms/)
231
- expect(result.error).toMatch(/3 handoffs still running/)
232
- expect(world.state.recreateImages).toEqual([])
233
- expect(world.state.image).toBe(CURRENT_IMAGE)
234
- } finally {
235
- await fs.rm(world.projectDirectory, {force: true, recursive: true})
236
- }
237
- })
238
-
239
- test("in-flight work that settles after a retry lets the roll continue", async () => {
240
- let attempts = 0
241
- const world = await createDockerWorld({
242
- image: CURRENT_IMAGE,
243
- shell: (command) => {
244
- if (!command.startsWith("await")) return {code: 0, signal: null, stderr: "", stdout: ""}
245
-
246
- attempts += 1
247
-
248
- return attempts < 3
249
- ? {code: 1, signal: null, stderr: "1 handoff still running", stdout: ""}
250
- : {code: 0, signal: null, stderr: "", stdout: ""}
251
- }
252
- })
253
- const config = buildConfig(world.projectDirectory, {inFlight: {command: "await in-flight {{service}}", intervalMs: 20, timeoutMs: 500}})
254
-
255
- try {
256
- const result = await rollComposeUnit({config, runCommand: world.runCommand, unit: config.compose[0]})
257
-
258
- expect(result.status).toBe("rolled")
259
- expect(attempts).toBe(3)
260
- expect(world.state.recreateImages).toEqual([TARGET_IMAGE])
261
- } finally {
262
- await fs.rm(world.projectDirectory, {force: true, recursive: true})
263
- }
264
- })
265
-
266
- test("a refused recreate fails the roll and leaves the previous container serving", async () => {
267
- const world = await createDockerWorld({image: CURRENT_IMAGE, recreateError: "Error response from daemon: No such image: registry.local/gateway:2.0.0"})
268
- const config = buildConfig(world.projectDirectory)
269
- const unit = config.compose[0]
270
-
271
- try {
272
- const result = await rollComposeUnit({config, runCommand: world.runCommand, unit})
273
-
274
- expect(result.status).toBe("failed")
275
- expect(result.stage).toBe("recreate")
276
- expect(result.image).toBe(CURRENT_IMAGE)
277
- expect(result.previousImage).toBe(CURRENT_IMAGE)
278
- expect(result.rollback).toBeUndefined()
279
- expect(result.error).toMatch(/Compose recreate failed for unit "gateway" \(image registry\.local\/gateway:2\.0\.0, previous registry\.local\/gateway:1\.4\.0\)/)
280
- expect(result.error).toMatch(/No such image/)
281
- expect(world.state.recreateImages).toEqual([])
282
- expect(world.state.image).toBe(CURRENT_IMAGE)
283
- expect(world.state.containerId).toBe("container-1")
284
- expect(world.state.state).toBe("running")
285
- expect(world.calls.filter((call) => call.args?.includes("up"))).toHaveLength(1)
286
- expect((await readComposeLedger({config, unit})).generations).toEqual([])
287
- expect(await fs.stat(path.join(world.projectDirectory, ".rollbridge-gateway.override.yml")).then(() => true, () => false)).toBe(false)
288
- } finally {
289
- await fs.rm(world.projectDirectory, {force: true, recursive: true})
290
- }
291
- })
292
-
293
- test("a candidate image that never activates is rolled back to the previous image", async () => {
294
- const world = await createDockerWorld({healthFor: (image) => image === TARGET_IMAGE ? "unhealthy" : "healthy", image: CURRENT_IMAGE})
295
- const config = buildConfig(world.projectDirectory, {activation: {intervalMs: 20, timeoutMs: 120}})
296
-
297
- try {
298
- const result = await rollComposeUnit({config, runCommand: world.runCommand, unit: config.compose[0]})
299
-
300
- expect(result.status).toBe("rolled_back")
301
- expect(result.stage).toBe("rollback")
302
- expect(result.error).toMatch(/did not activate within 120ms/)
303
- expect(result.rollback).toMatchObject({image: CURRENT_IMAGE, status: "rolled_back"})
304
- expect(world.state.recreateImages).toEqual([TARGET_IMAGE, CURRENT_IMAGE])
305
- expect(world.state.image).toBe(CURRENT_IMAGE)
306
- expect(world.state.health).toBe("healthy")
307
- expect(result.generations.map((generation) => [generation.image, generation.status])).toEqual([[CURRENT_IMAGE, "rolled_back"]])
308
- expect(result.generations[0].previousImage).toBe(TARGET_IMAGE)
309
- } finally {
310
- await fs.rm(world.projectDirectory, {force: true, recursive: true})
311
- }
312
- })
313
-
314
- test("a failed rollback is reported as a failed roll instead of a success", async () => {
315
- const world = await createDockerWorld({healthFor: () => "unhealthy", image: CURRENT_IMAGE})
316
- const config = buildConfig(world.projectDirectory, {activation: {intervalMs: 20, timeoutMs: 100}})
317
- const unit = config.compose[0]
318
-
319
- try {
320
- const result = await rollComposeUnit({config, runCommand: world.runCommand, unit})
321
-
322
- expect(result.status).toBe("failed")
323
- expect(result.stage).toBe("rollback_failed")
324
- expect(result.error).toMatch(/rollback to registry\.local\/gateway:1\.4\.0 did not activate/)
325
- expect((await readComposeLedger({config, unit})).generations).toEqual([])
326
- } finally {
327
- await fs.rm(world.projectDirectory, {force: true, recursive: true})
328
- }
329
- })
330
-
331
- test("compose-rollback selects the retained previous image", async () => {
332
- const world = await createDockerWorld({image: OLD_IMAGE})
333
- const config = buildConfig(world.projectDirectory)
334
- const unit = config.compose[0]
335
-
336
- try {
337
- await rollComposeUnit({config, runCommand: world.runCommand, unit})
338
-
339
- const result = await rollbackComposeUnit({config, runCommand: world.runCommand, unit})
340
-
341
- expect(result.status).toBe("rolled")
342
- expect(result.image).toBe(OLD_IMAGE)
343
- expect(result.previousImage).toBe(TARGET_IMAGE)
344
- expect(world.state.recreateImages).toEqual([TARGET_IMAGE, OLD_IMAGE])
345
- expect(world.state.image).toBe(OLD_IMAGE)
346
- } finally {
347
- await fs.rm(world.projectDirectory, {force: true, recursive: true})
348
- }
349
- })
350
-
351
- test("compose-rollback refuses when no earlier image is retained", async () => {
352
- const world = await createDockerWorld({image: CURRENT_IMAGE})
353
- const config = buildConfig(world.projectDirectory)
354
-
355
- try {
356
- await expect(rollbackComposeUnit({config, runCommand: world.runCommand, unit: config.compose[0]})).rejects.toThrow(/No earlier image is retained for Compose unit "gateway"/)
357
- } finally {
358
- await fs.rm(world.projectDirectory, {force: true, recursive: true})
359
- }
360
- })
361
-
362
- test("refuses to roll a unit whose container publishes the Rollbridge proxy port", async () => {
363
- const world = await createDockerWorld({image: CURRENT_IMAGE, ports: [{HostPort: "8182", containerPort: "8182"}]})
364
- const config = buildConfig(world.projectDirectory)
365
-
366
- try {
367
- await expect(rollComposeUnit({config, runCommand: world.runCommand, unit: config.compose[0]})).rejects.toThrow(/publishes port 8182, the Rollbridge proxy port/)
368
- expect(world.state.recreateImages).toEqual([])
369
- } finally {
370
- await fs.rm(world.projectDirectory, {force: true, recursive: true})
371
- }
372
- })
373
-
374
- test("refuses to roll while a release generation transition is unresolved", async () => {
375
- const world = await createDockerWorld({image: CURRENT_IMAGE})
376
- const config = buildConfig(world.projectDirectory)
377
-
378
- try {
379
- await expect(rollComposeUnit({
380
- config,
381
- inspectSocket: async () => ({activeReleaseId: "v1", alive: true, application: "compose-test"}),
382
- runCommand: world.runCommand,
383
- sendCommand: async () => ({
384
- application: "compose-test",
385
- generationTransition: {candidateReleaseId: "v2", phase: "activating_candidate", previousReleaseId: "v1"},
386
- status: "success"
387
- }),
388
- unit: config.compose[0]
389
- })).rejects.toThrow(/while release generation transition v2 is unresolved at activating_candidate/)
390
-
391
- expect(world.state.recreateImages).toEqual([])
392
- } finally {
393
- await fs.rm(world.projectDirectory, {force: true, recursive: true})
394
- }
395
- })
396
-
397
- test("compose-status reports the live container and the retained generations", async () => {
398
- const world = await createDockerWorld({image: CURRENT_IMAGE})
399
- const config = buildConfig(world.projectDirectory)
400
- const unit = config.compose[0]
401
-
402
- try {
403
- await rollComposeUnit({config, runCommand: world.runCommand, unit})
404
-
405
- const status = await composeUnitStatus({config, runCommand: world.runCommand, unit})
406
-
407
- expect(status.unitId).toBe("gateway")
408
- expect(status.container).toMatchObject({containerId: "container-2", healthStatus: "healthy", image: TARGET_IMAGE, state: "running"})
409
- expect(status.generations.map((generation) => generation.image)).toEqual([TARGET_IMAGE])
410
- expect(status.ledgerPath).toBe(path.join(world.projectDirectory, "compose.state.json"))
411
- } finally {
412
- await fs.rm(world.projectDirectory, {force: true, recursive: true})
413
- }
414
- })
415
- })