shared-hosting-sync 0.1.4 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +3 -0
- package/CHANGELOG.md +12 -0
- package/README.md +42 -8
- package/RELEASING.md +7 -7
- package/package.json +2 -2
- package/src/arguments.js +1 -1
- package/src/cli.js +9 -5
- package/src/core/deployment.js +1 -0
- package/src/core/envSchema.js +9 -0
- package/src/server/deployments.js +18 -4
- package/src/transport/ftp.js +1 -1
- package/src/transport/reliable.js +3 -3
- package/src/transport/sftp.js +8 -2
- package/src/ui/deployments.js +4 -2
- package/src/ui/style.css +2 -0
- package/src/ui/ui.js +1 -1
- package/src/uploadQueue.js +106 -0
- package/src/uploader.js +36 -7
- package/src/watcher.js +7 -2
package/.env.example
CHANGED
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,17 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.1.5 — 18 November 2026
|
|
4
|
+
|
|
5
|
+
- List server folders before comparing local files during an initial scan, avoiding
|
|
6
|
+
a separate server metadata request for every local file. Verified matches are
|
|
7
|
+
saved to SQLite before the next sync.
|
|
8
|
+
- Upload through three independent connections by default, balanced by file size.
|
|
9
|
+
Idle workers take pending work from other queues.
|
|
10
|
+
- Return interrupted files to the pending queue for healthy workers, with bounded
|
|
11
|
+
retries and completion-based progress. Failed files remain pending in SQLite.
|
|
12
|
+
- Show individual queue status in the dashboard and allow 1–8 parallel uploads
|
|
13
|
+
through project settings, DEVSYNC_CONCURRENCY, or --concurrency.
|
|
14
|
+
|
|
3
15
|
## 0.1.4 — 27 October 2026
|
|
4
16
|
|
|
5
17
|
- Use persistent per-file sync records for fast local size and timestamp checks.
|
package/README.md
CHANGED
|
@@ -6,8 +6,7 @@ choose a local folder and a server destination, then deploy or keep files in syn
|
|
|
6
6
|
Use the terminal or a local browser dashboard. Both interfaces control the same
|
|
7
7
|
projects, settings, and deployment operations.
|
|
8
8
|
|
|
9
|
-
>
|
|
10
|
-
> CLI and UI onboarding, previews, deployment, sync, and watching are implemented.
|
|
9
|
+
> Requires Node.js 24 or newer.
|
|
11
10
|
|
|
12
11
|
The package name is **shared-hosting-sync**. Run it as `shared-hosting-sync` or
|
|
13
12
|
use the shorter `devsync` alias shown in examples below. Both commands run the
|
|
@@ -44,10 +43,11 @@ shared-hosting-sync --version
|
|
|
44
43
|
shared-hosting-sync ui
|
|
45
44
|
```
|
|
46
45
|
|
|
47
|
-
Alternatively,
|
|
46
|
+
Alternatively, run `npm run release:check` from the checkout and install the resulting
|
|
47
|
+
archive. Replace `<version>` with its actual version:
|
|
48
48
|
|
|
49
49
|
```sh
|
|
50
|
-
npm install -g ./artifacts/shared-hosting-sync
|
|
50
|
+
npm install -g "./artifacts/shared-hosting-sync-<version>.tgz"
|
|
51
51
|
```
|
|
52
52
|
|
|
53
53
|
For development, install from this checkout:
|
|
@@ -67,12 +67,46 @@ name conflict:
|
|
|
67
67
|
|
|
68
68
|
```sh
|
|
69
69
|
npm uninstall -g devsync
|
|
70
|
-
npm install -g
|
|
70
|
+
npm install -g shared-hosting-sync
|
|
71
71
|
```
|
|
72
72
|
|
|
73
73
|
This replaces the old global installation. Saved projects and credentials remain
|
|
74
|
-
in `~/.devsync/` (or your configured `DEVSYNC_HOME`).
|
|
75
|
-
|
|
74
|
+
in `~/.devsync/` (or your configured `DEVSYNC_HOME`). To back up the settings,
|
|
75
|
+
stop DevSync and copy that directory somewhere private. Keep the database and
|
|
76
|
+
`master.key` together so saved credentials can be restored.
|
|
77
|
+
|
|
78
|
+
## First sync for an existing site
|
|
79
|
+
|
|
80
|
+
1. Choose **Scan server / Initial sync** and confirm. DevSync lists the destination
|
|
81
|
+
folders first, then checks local files against that inventory. Files with the
|
|
82
|
+
same size are checked by content before being marked as synced.
|
|
83
|
+
2. Review pending files, then choose **Sync changes** to upload them. The scan
|
|
84
|
+
saves verified matches to SQLite and does not upload or delete server files.
|
|
85
|
+
3. For everyday updates, use **Sync changes** directly. It uses saved records and
|
|
86
|
+
only queues new or changed files. For an empty destination, skip the scan and
|
|
87
|
+
use **Sync changes** or **Deploy all files**.
|
|
88
|
+
|
|
89
|
+
## Parallel uploads
|
|
90
|
+
|
|
91
|
+
Deploy, Sync, and Watch use up to **3 upload queues** by default. Each queue owns
|
|
92
|
+
an independent connection to the same host and configured port, so FTP commands
|
|
93
|
+
never overlap on one control connection. FTP/FTPS data connections still use the
|
|
94
|
+
server's passive ports. SFTP uses separate SSH/SFTP sessions.
|
|
95
|
+
|
|
96
|
+
Files are initially balanced by byte size. Idle queues take pending work from
|
|
97
|
+
other queues. If a connection drops, its interrupted file returns to the pending
|
|
98
|
+
list and healthy queues continue. A file gets at most two retries after its first
|
|
99
|
+
attempt; replacement connection rounds are also bounded. Permission errors are
|
|
100
|
+
reported immediately. Failed files remain pending for a later sync. An uncertain
|
|
101
|
+
rename retries the entire temporary-file upload.
|
|
102
|
+
|
|
103
|
+
Set **Parallel uploads** under Edit setup → Hosting → SSH key and advanced
|
|
104
|
+
settings, or use `DEVSYNC_CONCURRENCY=3` in the project's environment file.
|
|
105
|
+
The CLI also accepts `devsync sync [label] --concurrency 3`. Values from 1 to 8
|
|
106
|
+
are supported; choose 1 for restrictive hosting accounts. Small batches open only
|
|
107
|
+
as many connections as needed, and previews or unchanged syncs open none.
|
|
108
|
+
Stopping Watch finishes the current upload batch, including bounded retries,
|
|
109
|
+
then closes its connections. Progress counts completed files, never retry attempts.
|
|
76
110
|
|
|
77
111
|
## Start from a project folder
|
|
78
112
|
|
|
@@ -350,7 +384,7 @@ Provision any server-side application secrets separately.
|
|
|
350
384
|
- Records belong to a source folder and destination (protocol, host, port, account,
|
|
351
385
|
and server folder). Changing either starts a separate baseline.
|
|
352
386
|
- If files already exist on the server, use **Scan server / Initial sync** in the
|
|
353
|
-
dashboard once before your first sync. It asks for confirmation, compares server
|
|
387
|
+
dashboard once before your first sync. It asks for confirmation, lists server folders first, then compares local and server
|
|
354
388
|
content using SHA-256 for equal-size files, and saves only matching files. It
|
|
355
389
|
uploads or deletes nothing; use Sync changes afterward for pending files.
|
|
356
390
|
Terminal equivalent: `devsync scan [label]` (or `--confirm` for automation).
|
package/RELEASING.md
CHANGED
|
@@ -24,7 +24,7 @@ aliases, SQLite storage, environment onboarding, real FTP deploy/sync, dashboard
|
|
|
24
24
|
startup, API identity, and UI assets must pass. Nothing is installed into the
|
|
25
25
|
developer's usual global prefix. Generated files are under `artifacts/`:
|
|
26
26
|
|
|
27
|
-
- `shared-hosting-sync
|
|
27
|
+
- `shared-hosting-sync-<version>.tgz` (version follows package.json)
|
|
28
28
|
- `release.json`, including the complete package file list
|
|
29
29
|
- `SHA256SUMS`
|
|
30
30
|
|
|
@@ -45,9 +45,9 @@ npm whoami
|
|
|
45
45
|
npm view shared-hosting-sync name version
|
|
46
46
|
```
|
|
47
47
|
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
the
|
|
48
|
+
Confirm the signed-in account is the intended maintainer and that the candidate
|
|
49
|
+
version has not already been published. Replace `<version>` in the examples with
|
|
50
|
+
the candidate version from package.json. Use npm's browser sign-in and
|
|
51
51
|
complete any account/2FA requirements personally; do not put tokens into source
|
|
52
52
|
files or chat.
|
|
53
53
|
|
|
@@ -55,7 +55,7 @@ Once the account, version, passing CI, package manifest, and checksum are review
|
|
|
55
55
|
publish the exact checked archive:
|
|
56
56
|
|
|
57
57
|
```sh
|
|
58
|
-
npm publish ./artifacts/shared-hosting-sync
|
|
58
|
+
npm publish "./artifacts/shared-hosting-sync-<version>.tgz" --access public
|
|
59
59
|
```
|
|
60
60
|
|
|
61
61
|
Publishing with `--access public` makes the package contents publicly available. Publishing
|
|
@@ -66,8 +66,8 @@ published version; fix issues in a new version.
|
|
|
66
66
|
## Verify publication
|
|
67
67
|
|
|
68
68
|
```sh
|
|
69
|
-
npm view shared-hosting-sync
|
|
70
|
-
npm install -g shared-hosting-sync
|
|
69
|
+
npm view "shared-hosting-sync@<version>" name version dist.integrity
|
|
70
|
+
npm install -g "shared-hosting-sync@<version>"
|
|
71
71
|
shared-hosting-sync --version
|
|
72
72
|
devsync --version
|
|
73
73
|
shared-hosting-sync ui
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "shared-hosting-sync",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.5",
|
|
4
4
|
"description": "Deploy and sync local projects to shared hosting over FTP, FTPS, or SFTP, with a CLI and local dashboard.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
|
@@ -49,7 +49,7 @@
|
|
|
49
49
|
"scripts": {
|
|
50
50
|
"start": "node src/cli.js",
|
|
51
51
|
"release:check": "node scripts/release-check.js",
|
|
52
|
-
"test": "node test/ignore.test.js && node --test --test-timeout=60000 --test-concurrency=1 test/uploader.test.js test/sync-state.test.js test/transport-ftp.test.js test/cli-deployment.test.js test/cli-onboarding.test.js test/onboarding.test.js test/ui-onboarding.test.js test/ui-deployments.test.js test/ui-sync.test.js test/reliability.test.js test/protocols.test.js test/transport-sftp.test.js test/watcher.test.js test/envResolver.test.js test/db.test.js test/secrets.test.js",
|
|
52
|
+
"test": "node test/ignore.test.js && node --test --test-timeout=60000 --test-concurrency=1 test/uploader.test.js test/upload-queue.test.js test/sync-state.test.js test/transport-ftp.test.js test/cli-deployment.test.js test/cli-onboarding.test.js test/onboarding.test.js test/ui-onboarding.test.js test/ui-deployments.test.js test/ui-sync.test.js test/reliability.test.js test/protocols.test.js test/transport-sftp.test.js test/watcher.test.js test/envResolver.test.js test/db.test.js test/secrets.test.js",
|
|
53
53
|
"prepack": "npm test",
|
|
54
54
|
"prepublishOnly": "npm test"
|
|
55
55
|
},
|
package/src/arguments.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
const VALUE_FLAGS = new Set(['transport', 'env-file', 'remote-path', 'host', 'username', 'port', 'label', 'path'])
|
|
1
|
+
const VALUE_FLAGS = new Set(['concurrency', 'transport', 'env-file', 'remote-path', 'host', 'username', 'port', 'label', 'path'])
|
|
2
2
|
const BOOL_FLAGS = new Set(['confirm', 'dry-run', 'force', 'plan', 'sync', 'edit', 'browse', 'browser', 'help', 'version'])
|
|
3
3
|
|
|
4
4
|
export function parseArgs(argv) {
|
package/src/cli.js
CHANGED
|
@@ -64,7 +64,7 @@ try {
|
|
|
64
64
|
const sessionOverrides = new Map()
|
|
65
65
|
const prompt = createPrompter()
|
|
66
66
|
const flagOverrides = () => Object.fromEntries([
|
|
67
|
-
['transport', 'DEVSYNC_TRANSPORT'], ['host', 'DEVSYNC_HOST'],
|
|
67
|
+
['concurrency', 'DEVSYNC_CONCURRENCY'], ['transport', 'DEVSYNC_TRANSPORT'], ['host', 'DEVSYNC_HOST'],
|
|
68
68
|
['username', 'DEVSYNC_USERNAME'], ['port', 'DEVSYNC_PORT'], ['remote-path', 'DEVSYNC_REMOTE_PATH']
|
|
69
69
|
].filter(([flag]) => flags[flag] !== undefined).map(([flag, key]) => [key, flags[flag]]))
|
|
70
70
|
|
|
@@ -137,7 +137,7 @@ async function credsForProject(project, { deployment = false } = {}) {
|
|
|
137
137
|
envFile: flags['env-file'], overrides: { ...flagOverrides(), ...sessionOverrides.get(project.id) }
|
|
138
138
|
})
|
|
139
139
|
Object.assign(project, context.project)
|
|
140
|
-
return { ...context.creds, dryRun: context.dryRun, debounceMs: context.debounceMs }
|
|
140
|
+
return { ...context.creds, dryRun: context.dryRun, debounceMs: context.debounceMs, concurrency: context.concurrency }
|
|
141
141
|
}
|
|
142
142
|
const r = resolveEnv(project.id, {
|
|
143
143
|
envFile: flags['env-file'], overrides: { ...flagOverrides(), ...sessionOverrides.get(project.id) }
|
|
@@ -331,11 +331,12 @@ const COMMANDS = {
|
|
|
331
331
|
setStatus(proj.id, 'idle')
|
|
332
332
|
return
|
|
333
333
|
}
|
|
334
|
-
t = await createTransport(creds, undefined, { onRetry: retryNotice })
|
|
335
334
|
let lastPct = -1
|
|
336
335
|
const r = await runPlan(p, t, {
|
|
337
|
-
dryRun,
|
|
338
|
-
|
|
336
|
+
dryRun, concurrency: creds.concurrency,
|
|
337
|
+
createWorker: () => createTransport(creds, undefined, { retries: 0 }),
|
|
338
|
+
onRetry: event => warn(`Queue ${event.workerId}: ${event.connectionFailure ? 'connection unavailable' : event.item.rel + ' interrupted'}; returned to the pending queue.`),
|
|
339
|
+
onFileResult: ({ index, total, item }) => {
|
|
339
340
|
const pct = Math.floor(((index + 1) / total) * 100)
|
|
340
341
|
if (pct !== lastPct && pct % 10 === 0) {
|
|
341
342
|
lastPct = pct
|
|
@@ -400,6 +401,8 @@ const COMMANDS = {
|
|
|
400
401
|
const w = new ProjectWatcher(proj, matcher, t, {
|
|
401
402
|
dryRun: creds.dryRun || !!flags['dry-run'],
|
|
402
403
|
debounceMs: creds.debounceMs,
|
|
404
|
+
concurrency: creds.concurrency,
|
|
405
|
+
createWorker: () => createTransport(creds, flags.transport, { retries: 0 }),
|
|
403
406
|
onEvent: (e) => {
|
|
404
407
|
if (e.type === 'error') {
|
|
405
408
|
setStatus(proj.id, 'error', { error: e.message })
|
|
@@ -503,6 +506,7 @@ ${C.b('options')}
|
|
|
503
506
|
--edit setup: edit connection settings
|
|
504
507
|
--browse setup: choose a different server folder
|
|
505
508
|
--label <name> name a new project
|
|
509
|
+
--concurrency <1-8> parallel upload queues (default: 3)
|
|
506
510
|
--confirm confirm a first-time server scan
|
|
507
511
|
--dry-run --force preview / include unchanged files
|
|
508
512
|
|
package/src/core/deployment.js
CHANGED
|
@@ -18,6 +18,7 @@ export function resolveDeployment(project, options = {}) {
|
|
|
18
18
|
matcher: buildMatcher(target.localPath),
|
|
19
19
|
creds: credsFromResolved(resolved.values),
|
|
20
20
|
dryRun: ['true', '1', true].includes(resolved.values.DEVSYNC_DRY_RUN),
|
|
21
|
+
concurrency: Number(resolved.values.DEVSYNC_CONCURRENCY),
|
|
21
22
|
debounceMs: Number(resolved.values.DEVSYNC_DEBOUNCE_MS),
|
|
22
23
|
values: resolved.values
|
|
23
24
|
}
|
package/src/core/envSchema.js
CHANGED
|
@@ -82,6 +82,15 @@ export const ENV_SCHEMA = [
|
|
|
82
82
|
label: 'Dry run',
|
|
83
83
|
help: 'Plan and report without uploading.'
|
|
84
84
|
},
|
|
85
|
+
{
|
|
86
|
+
key: 'DEVSYNC_CONCURRENCY',
|
|
87
|
+
type: 'int',
|
|
88
|
+
default: 3,
|
|
89
|
+
min: 1,
|
|
90
|
+
max: 8,
|
|
91
|
+
label: 'Parallel uploads',
|
|
92
|
+
help: 'Upload queues (1–8). Default 3. Use 1 if your host limits connections.'
|
|
93
|
+
},
|
|
85
94
|
{
|
|
86
95
|
key: 'DEVSYNC_DEBOUNCE_MS',
|
|
87
96
|
type: 'int',
|
|
@@ -60,7 +60,7 @@ export class DeploymentService {
|
|
|
60
60
|
}
|
|
61
61
|
|
|
62
62
|
progress(record, event) {
|
|
63
|
-
if (!record.stopRequested) record.view.phase = event.phase === 'planning' ? 'planning' : 'uploading'
|
|
63
|
+
if (!record.stopRequested) record.view.phase = event.phase === 'inventory' ? 'inventory' : event.phase === 'planning' ? 'planning' : 'uploading'
|
|
64
64
|
record.view.progress = {
|
|
65
65
|
current: event.index, total: event.total, file: this.clean(record, event.item.rel),
|
|
66
66
|
bytes: record.view.progress?.bytes || 0
|
|
@@ -69,6 +69,7 @@ export class DeploymentService {
|
|
|
69
69
|
}
|
|
70
70
|
|
|
71
71
|
plan(record, plan) {
|
|
72
|
+
record.view.workers = []
|
|
72
73
|
record.view.plan = {
|
|
73
74
|
considered: plan.totalFiles, uploadCount: plan.uploads.length, skippedCount: plan.skipped.length,
|
|
74
75
|
bytes: plan.comparedBytes,
|
|
@@ -82,6 +83,15 @@ export class DeploymentService {
|
|
|
82
83
|
this.log(record, `Compared ${plan.totalFiles} files: ${plan.uploads.length} to upload, ${plan.skipped.length} unchanged`)
|
|
83
84
|
}
|
|
84
85
|
|
|
86
|
+
workerState(record, event) {
|
|
87
|
+
record.view.workers ||= []
|
|
88
|
+
const current = record.view.workers.findIndex(worker => worker.workerId === event.workerId)
|
|
89
|
+
const value = { workerId: event.workerId, state: event.state, file: event.file ? this.clean(record, event.file) : null }
|
|
90
|
+
if (current < 0) record.view.workers.push(value)
|
|
91
|
+
else record.view.workers[current] = value
|
|
92
|
+
record.view.updatedAt = stamp()
|
|
93
|
+
}
|
|
94
|
+
|
|
85
95
|
fileResult(record, event) {
|
|
86
96
|
record.view.progress = { current: event.index + 1, total: event.total, file: this.clean(record, event.item.rel), bytes: event.bytes }
|
|
87
97
|
this.log(record, event.success ? `${event.dryRun ? 'Would upload' : 'Uploaded'} ${event.item.rel}` : `Failed ${event.item.rel}: ${event.error}`, event.success ? 'info' : 'error')
|
|
@@ -125,7 +135,7 @@ export class DeploymentService {
|
|
|
125
135
|
projectId: project.id, operationId: crypto.randomUUID(), kind,
|
|
126
136
|
mode: kind === 'preview' ? options.mode || 'sync' : kind === 'deploy' ? 'deploy' : 'sync',
|
|
127
137
|
state: 'running', phase: 'connecting', dryRun, plan: null, result: null, progress: null,
|
|
128
|
-
logs: [], lastError: null, startedAt: stamp(), updatedAt: stamp(), remotePath: context.project.remotePath
|
|
138
|
+
logs: [], workers: [], concurrency: context.concurrency, lastError: null, startedAt: stamp(), updatedAt: stamp(), remotePath: context.project.remotePath
|
|
129
139
|
}
|
|
130
140
|
}
|
|
131
141
|
this.records.set(project.id, record)
|
|
@@ -151,6 +161,7 @@ export class DeploymentService {
|
|
|
151
161
|
}
|
|
152
162
|
if (record.stopRequested || this.closing) return
|
|
153
163
|
const context = record.context
|
|
164
|
+
const createWorker = () => this.connect(context.creds, undefined, { retries: 0 })
|
|
154
165
|
if (record.view.kind === 'scan') {
|
|
155
166
|
await connect()
|
|
156
167
|
const plan = await scanServer(context.project, context.matcher, record.transport, { onProgress: (event) => this.progress(record, event) })
|
|
@@ -168,6 +179,8 @@ export class DeploymentService {
|
|
|
168
179
|
if (record.stopRequested || this.closing) return
|
|
169
180
|
const watcher = this.watcherFactory(context.project, context.matcher, record.transport, {
|
|
170
181
|
dryRun: record.view.dryRun, debounceMs: context.debounceMs,
|
|
182
|
+
createWorker, concurrency: context.concurrency,
|
|
183
|
+
onWorkerState: event => this.workerState(record, event),
|
|
171
184
|
onProgress: (event) => this.progress(record, event),
|
|
172
185
|
onPlan: (plan) => this.plan(record, plan),
|
|
173
186
|
onFileResult: (event) => this.fileResult(record, event),
|
|
@@ -193,9 +206,10 @@ export class DeploymentService {
|
|
|
193
206
|
force: record.view.mode === 'deploy', onProgress: (event) => this.progress(record, event)
|
|
194
207
|
})
|
|
195
208
|
this.plan(record, plan)
|
|
196
|
-
if (!record.view.dryRun && plan.uploads.length) await connect()
|
|
197
209
|
const result = await runPlan(plan, record.transport, {
|
|
198
|
-
dryRun: record.view.dryRun,
|
|
210
|
+
dryRun: record.view.dryRun, createWorker, concurrency: context.concurrency,
|
|
211
|
+
onWorkerState: event => this.workerState(record, event),
|
|
212
|
+
onRetry: event => this.log(record, `Queue ${event.workerId}: ${event.connectionFailure ? 'connection unavailable' : event.item.rel + ' interrupted'}; returned to the pending queue.`, 'warning'),
|
|
199
213
|
onProgress: (event) => this.progress(record, event), onFileResult: (event) => this.fileResult(record, event)
|
|
200
214
|
})
|
|
201
215
|
this.complete(record, { done: result.done.length, failed: result.failed, bytes: result.bytes, dryRun: record.view.dryRun })
|
package/src/transport/ftp.js
CHANGED
|
@@ -35,7 +35,7 @@ export class FtpTransport {
|
|
|
35
35
|
const items = await this.client.list(remotePath || '/')
|
|
36
36
|
return items.map((e) => ({
|
|
37
37
|
name: e.name,
|
|
38
|
-
type: e.isDirectory ? 'dir' : 'file',
|
|
38
|
+
type: e.isDirectory ? 'dir' : e.isFile ? 'file' : 'other',
|
|
39
39
|
size: e.size ?? 0,
|
|
40
40
|
mtime: e.modifyTime ? new Date(e.modifyTime) : null
|
|
41
41
|
}))
|
|
@@ -19,15 +19,15 @@ export class ReliableTransport {
|
|
|
19
19
|
|
|
20
20
|
async connect() { await this.transaction(async () => {}); return this }
|
|
21
21
|
|
|
22
|
-
async transaction(action) {
|
|
22
|
+
async transaction(action, { retries = this.retries } = {}) {
|
|
23
23
|
for (let attempt = 0; ; attempt++) {
|
|
24
24
|
if (this.closed) throw new Error('Transport has been closed')
|
|
25
25
|
try {
|
|
26
26
|
this.raw ||= await this.connectRaw()
|
|
27
27
|
return await action(this.raw)
|
|
28
28
|
} catch (error) {
|
|
29
|
-
if (!isTransient(error) || attempt >=
|
|
30
|
-
this.onRetry({ attempt: attempt + 1, retries
|
|
29
|
+
if (!isTransient(error) || attempt >= retries || this.closed) throw error
|
|
30
|
+
this.onRetry({ attempt: attempt + 1, retries })
|
|
31
31
|
if (this.raw) await this.raw.close().catch(() => {})
|
|
32
32
|
this.raw = null
|
|
33
33
|
await delay(this.retryDelayMs * 2 ** attempt)
|
package/src/transport/sftp.js
CHANGED
|
@@ -40,7 +40,7 @@ export class SftpTransport {
|
|
|
40
40
|
.filter((e) => e.name !== '.' && e.name !== '..')
|
|
41
41
|
.map((e) => ({
|
|
42
42
|
name: e.name,
|
|
43
|
-
type: e.type === 'd' ? 'dir' : 'file',
|
|
43
|
+
type: e.type === 'd' ? 'dir' : e.type === '-' ? 'file' : 'other',
|
|
44
44
|
size: e.size ?? 0,
|
|
45
45
|
mtime: e.modifyTime ? new Date(e.modifyTime) : null
|
|
46
46
|
}))
|
|
@@ -50,7 +50,13 @@ export class SftpTransport {
|
|
|
50
50
|
if (!remotePath) return
|
|
51
51
|
const type = await this.client.exists(remotePath)
|
|
52
52
|
if (type && type !== 'd') throw new Error(`Remote directory path is occupied by a file: ${remotePath}`)
|
|
53
|
-
if (!type)
|
|
53
|
+
if (!type) {
|
|
54
|
+
try { await this.client.mkdir(remotePath, true) }
|
|
55
|
+
catch (error) {
|
|
56
|
+
// Another upload worker may have created the same directory meanwhile.
|
|
57
|
+
if (await this.client.exists(remotePath) !== 'd') throw error
|
|
58
|
+
}
|
|
59
|
+
}
|
|
54
60
|
}
|
|
55
61
|
|
|
56
62
|
async exists(remotePath) {
|
package/src/ui/deployments.js
CHANGED
|
@@ -24,7 +24,7 @@ export class DeploymentPanel {
|
|
|
24
24
|
this.settingsDryRun = ['true', '1', true].includes(configuration.values.DEVSYNC_DRY_RUN)
|
|
25
25
|
this.configured = !Object.keys(configuration.errors).length
|
|
26
26
|
const element = document.querySelector('#deploymentPanel')
|
|
27
|
-
element.innerHTML = `<h2>Deploy your project</h2><p class="muted">Sync checks local file sizes and timestamps against your saved sync records. Deploy uploads all eligible files. Server files are never deleted.</p><div class="operation-options"><div class="field"><label for="previewMode">Preview mode</label><select id="previewMode"><option value="sync">Changed files (sync)</option><option value="deploy">All eligible files (deploy)</option></select></div><label class="check"><input id="operationDryRun" type="checkbox" ${this.settingsDryRun ? 'checked disabled' : ''}>Preview only for deploy, sync and watch</label></div>${this.settingsDryRun ? '<p class="help">Preview-only mode is enabled in this project’s settings.</p>' : ''}<div class="operation-actions"><button class="secondary" id="previewUploads">Preview uploads</button><button class="secondary" id="deployFiles">Deploy all files</button><button class="primary" id="syncFiles">Sync changes</button><button class="secondary" id="startWatching">Start watching</button><button class="danger" id="stopWatching" hidden>Stop watching</button></div><div class="sync-baseline"><p id="syncBaseline" class="help"></p><button class="secondary" id="scanServer">Scan server / Initial sync</button><p class="help">Use this button only if you are syncing for the first time. It
|
|
27
|
+
element.innerHTML = `<h2>Deploy your project</h2><p class="muted">Sync checks local file sizes and timestamps against your saved sync records. Deploy uploads all eligible files. Server files are never deleted. Up to ${Number(configuration.values.DEVSYNC_CONCURRENCY) || 3} upload queues share the work and retry interrupted files.</p><div class="operation-options"><div class="field"><label for="previewMode">Preview mode</label><select id="previewMode"><option value="sync">Changed files (sync)</option><option value="deploy">All eligible files (deploy)</option></select></div><label class="check"><input id="operationDryRun" type="checkbox" ${this.settingsDryRun ? 'checked disabled' : ''}>Preview only for deploy, sync and watch</label></div>${this.settingsDryRun ? '<p class="help">Preview-only mode is enabled in this project’s settings.</p>' : ''}<div class="operation-actions"><button class="secondary" id="previewUploads">Preview uploads</button><button class="secondary" id="deployFiles">Deploy all files</button><button class="primary" id="syncFiles">Sync changes</button><button class="secondary" id="startWatching">Start watching</button><button class="danger" id="stopWatching" hidden>Stop watching</button></div><div class="sync-baseline"><p id="syncBaseline" class="help"></p><button class="secondary" id="scanServer">Scan server / Initial sync</button><p class="help">Use this button only if you are syncing for the first time. It lists server folders first, then checks local files and saves verified matches without uploading. After that, use Sync changes.</p></div><p class="help">Watch syncs current files first, then follows saves while devsync is running. Stop waits for the current batch to finish.</p><div id="operationStatus" class="notice" role="status"></div><div class="operation-progress"><progress id="uploadProgress" max="1" value="0" aria-label="Operation progress"></progress><span id="progressText" class="help"></span></div><div id="uploadQueues" class="upload-queues" aria-label="Upload queues"></div><div id="operationResult"></div><div id="operationPlan"></div><details id="activityDetails"><summary>Activity log</summary><ol class="operation-log" id="operationLogs"><li>No operations yet.</li></ol></details><p id="lastSync" class="help"></p>`
|
|
28
28
|
document.querySelector('#previewUploads').onclick = () => this.start('preview')
|
|
29
29
|
document.querySelector('#deployFiles').onclick = () => this.start('deploy')
|
|
30
30
|
document.querySelector('#syncFiles').onclick = () => this.start('sync')
|
|
@@ -93,7 +93,7 @@ export class DeploymentPanel {
|
|
|
93
93
|
document.querySelector('#stopWatching').hidden = !startingWatch
|
|
94
94
|
this.disable('stopWatching', !startingWatch || activity.stopping)
|
|
95
95
|
const status = document.querySelector('#operationStatus')
|
|
96
|
-
const phases = { connecting: 'Connecting to hosting…', planning: activity.kind === 'scan' ? '
|
|
96
|
+
const phases = { connecting: 'Connecting to hosting…', inventory: 'Listing server folders…', planning: activity.kind === 'scan' ? 'Comparing local files and saving verified matches…' : 'Checking local files against saved sync records…', uploading: activity.dryRun ? 'Building the preview…' : 'Uploading files…', watching: 'Watching for local saves.', stopping: 'Stopping watching — finishing the current batch…', complete: 'Operation complete.', error: 'Operation needs attention.', idle: 'Preview uploads before deploying, or start a sync when you are ready.' }
|
|
97
97
|
let text = activity.stopping ? phases.stopping : phases[activity.phase] || phases.idle
|
|
98
98
|
if (activity.state === 'success') text = activity.dryRun ? 'Preview complete. No files were uploaded.' : activity.result?.uploaded === 0 ? 'Up to date. No uploads needed.' : 'Upload complete.'
|
|
99
99
|
if (activity.kind === 'scan' && activity.state === 'success') text = `Server scan complete. ${activity.result?.matched || 0} files matched; ${activity.plan?.uploadCount || 0} new or changed. Use Sync changes to upload them.`
|
|
@@ -112,6 +112,8 @@ export class DeploymentPanel {
|
|
|
112
112
|
document.querySelector('#syncBaseline').textContent = baseline?.ready
|
|
113
113
|
? `${baseline.trackedFiles} files tracked for this source and destination. Sync changes uses these records.${baseline.scannedAt ? ' Last server scan: ' + new Date(baseline.scannedAt).toLocaleString() + '.' : ''}`
|
|
114
114
|
: 'No sync records yet for this source and destination. If files are already on your server, scan once to avoid re-uploading matches. Otherwise, Sync changes uploads all eligible files and starts tracking them.'
|
|
115
|
+
const queueNames = { connecting: 'Connecting', uploading: 'Uploading', failed: 'Unavailable · work returned to queue', complete: 'Finished' }
|
|
116
|
+
document.querySelector('#uploadQueues').innerHTML = (activity.workers || []).sort((a, b) => a.workerId - b.workerId).map(worker => `<div class="upload-queue queue-${esc(worker.state)}"><strong>Queue ${worker.workerId}</strong><span>${esc(queueNames[worker.state] || worker.state)}</span>${worker.file ? `<small title="${esc(worker.file)}">${esc(worker.file)}</small>` : ''}</div>`).join('')
|
|
115
117
|
const result = activity.result
|
|
116
118
|
document.querySelector('#operationResult').innerHTML = result?.scanned !== undefined ? `<div class="operation-metrics"><div><strong>${result.scanned}</strong><span>files scanned</span></div><div><strong>${result.matched}</strong><span>matching files</span></div><div><strong>${result.planned}</strong><span>pending upload</span></div></div>` : result ? `<div class="operation-metrics"><div><strong>${result.dryRun ? result.planned : result.uploaded}</strong><span>${result.dryRun ? 'would upload' : 'uploaded'}</span></div><div><strong>${result.failedCount ?? result.failed.length}</strong><span>failed</span></div><div><strong>${bytes(result.bytes)}</strong><span>uploaded</span></div></div>${result.failed.length ? `<details open><summary>Files needing attention</summary><ul class="errors">${result.failed.map((file) => `<li>${esc(file.rel)} — ${esc(file.error)}</li>`).join('')}</ul></details>` : ''}` : ''
|
|
117
119
|
this.renderPlan(activity.plan)
|
package/src/ui/style.css
CHANGED
|
@@ -145,3 +145,5 @@ h1{font-size:25px}
|
|
|
145
145
|
@media(max-width:580px){.brand{font-size:21px;gap:9px}.brand-logo{width:34px;height:34px}.topbar{gap:12px}.connection-pill{font-size:11px;padding:4px 9px}}
|
|
146
146
|
|
|
147
147
|
.about-brand{flex-wrap:wrap}.app-version{font-size:12px;line-height:1.5;font-weight:600;letter-spacing:0;color:var(--accent);background:#203e38;border:1px solid #33554e;border-radius:999px;padding:3px 10px}
|
|
148
|
+
|
|
149
|
+
.upload-queues{display:grid;grid-template-columns:repeat(auto-fit,minmax(160px,1fr));gap:10px;margin:16px 0}.upload-queues:empty{display:none}.upload-queue{min-width:0;padding:12px;border:1px solid var(--line);border-radius:8px;background:#10161f}.upload-queue strong,.upload-queue span,.upload-queue small{display:block}.upload-queue span{font-size:12px;color:var(--muted);margin-top:4px}.upload-queue small{overflow:hidden;text-overflow:ellipsis;white-space:nowrap;color:var(--muted);margin-top:6px}.queue-uploading{border-color:#397367}.queue-uploading strong{color:var(--accent)}.queue-failed span{color:var(--danger)}
|
package/src/ui/ui.js
CHANGED
|
@@ -122,7 +122,7 @@ function renderWizard() {
|
|
|
122
122
|
} else if (step === 2) {
|
|
123
123
|
const transport = draft.values.DEVSYNC_TRANSPORT
|
|
124
124
|
const loaded = Object.entries(draft.sources).filter(([, value]) => value === '.env' || value === '.devsync.env').length
|
|
125
|
-
content = `<h2>Connect your hosting account</h2><p>${loaded ? `Loaded ${loaded} hosting settings from this folder’s environment files. Fill in anything missing.` : 'Enter the connection details from your hosting provider.'} Imported passwords stay on this computer and are not sent to the browser.</p><form id="hostingForm" autocomplete="off"><div class="form-grid"><div class="field"><label for="DEVSYNC_TRANSPORT">Protocol${source('DEVSYNC_TRANSPORT')}</label><select id="DEVSYNC_TRANSPORT" name="DEVSYNC_TRANSPORT"><option value="">Choose a protocol</option>${['ftps', 'sftp', 'ftp'].map((value) => `<option value="${value}" ${transport === value ? 'selected' : ''}>${value === 'ftps' ? 'FTPS — encrypted FTP' : value === 'sftp' ? 'SFTP — SSH file transfer' : 'FTP — unencrypted'}</option>`).join('')}</select></div>${input('DEVSYNC_HOST', 'Host')}${input('DEVSYNC_PORT', 'Port', { type: 'number', optional: true })}${input('DEVSYNC_USERNAME', 'Username')}${input('DEVSYNC_PASSWORD', 'Password / key passphrase', { secret: true })}</div><details><summary>SSH key and advanced settings</summary><div class="form-grid">${input('DEVSYNC_PRIVATE_KEY', 'SSH private key path', { secret: true, optional: true })}<div class="field"><label for="DEVSYNC_DRY_RUN">Default operation</label><select id="DEVSYNC_DRY_RUN" name="DEVSYNC_DRY_RUN"><option value="false" ${String(draft.values.DEVSYNC_DRY_RUN) === 'false' ? 'selected' : ''}>Allow uploads when requested</option><option value="true" ${String(draft.values.DEVSYNC_DRY_RUN) === 'true' ? 'selected' : ''}>Preview only</option></select></div>${input('DEVSYNC_DEBOUNCE_MS', 'Watch delay (milliseconds)', { type: 'number', optional: true })}</div><p class="help">An SSH key is optional and applies to SFTP only. With a key, the password field is an optional passphrase. Local deletions are never mirrored.</p></details><div class="actions"><button class="secondary" type="button" id="backSetup">Back</button><button class="secondary" type="button" id="cancelSetup">Cancel</button><span class="spacer"></span><button class="primary" type="submit">Test connection & continue</button></div></form>`
|
|
125
|
+
content = `<h2>Connect your hosting account</h2><p>${loaded ? `Loaded ${loaded} hosting settings from this folder’s environment files. Fill in anything missing.` : 'Enter the connection details from your hosting provider.'} Imported passwords stay on this computer and are not sent to the browser.</p><form id="hostingForm" autocomplete="off"><div class="form-grid"><div class="field"><label for="DEVSYNC_TRANSPORT">Protocol${source('DEVSYNC_TRANSPORT')}</label><select id="DEVSYNC_TRANSPORT" name="DEVSYNC_TRANSPORT"><option value="">Choose a protocol</option>${['ftps', 'sftp', 'ftp'].map((value) => `<option value="${value}" ${transport === value ? 'selected' : ''}>${value === 'ftps' ? 'FTPS — encrypted FTP' : value === 'sftp' ? 'SFTP — SSH file transfer' : 'FTP — unencrypted'}</option>`).join('')}</select></div>${input('DEVSYNC_HOST', 'Host')}${input('DEVSYNC_PORT', 'Port', { type: 'number', optional: true })}${input('DEVSYNC_USERNAME', 'Username')}${input('DEVSYNC_PASSWORD', 'Password / key passphrase', { secret: true })}</div><details><summary>SSH key and advanced settings</summary><div class="form-grid">${input('DEVSYNC_PRIVATE_KEY', 'SSH private key path', { secret: true, optional: true })}<div class="field"><label for="DEVSYNC_DRY_RUN">Default operation</label><select id="DEVSYNC_DRY_RUN" name="DEVSYNC_DRY_RUN"><option value="false" ${String(draft.values.DEVSYNC_DRY_RUN) === 'false' ? 'selected' : ''}>Allow uploads when requested</option><option value="true" ${String(draft.values.DEVSYNC_DRY_RUN) === 'true' ? 'selected' : ''}>Preview only</option></select></div>${input('DEVSYNC_CONCURRENCY', 'Parallel uploads (1–8)', { type: 'number', optional: true })}${input('DEVSYNC_DEBOUNCE_MS', 'Watch delay (milliseconds)', { type: 'number', optional: true })}</div><p class="help">An SSH key is optional and applies to SFTP only. With a key, the password field is an optional passphrase. Parallel uploads defaults to 3; choose 1 if your host limits connections. Local deletions are never mirrored.</p></details><div class="actions"><button class="secondary" type="button" id="backSetup">Back</button><button class="secondary" type="button" id="cancelSetup">Cancel</button><span class="spacer"></span><button class="primary" type="submit">Test connection & continue</button></div></form>`
|
|
126
126
|
} else if (step === 3) {
|
|
127
127
|
const selected = draft.values.DEVSYNC_REMOTE_PATH
|
|
128
128
|
content = `<h2>Choose the folder on your server</h2><p>Open a folder, then select “Use server folder”. Your local files will be placed inside this destination.</p><div class="notice success">Connection verified with ${esc(draft.values.DEVSYNC_HOST)}.</div><div class="selection"><small>Selected destination ${source('DEVSYNC_REMOTE_PATH')}</small><strong id="selectedRemote">${esc(selected || 'Choose a server folder below')}</strong></div><div id="remotePicker"></div><div class="actions"><button class="secondary" type="button" id="backSetup">Back</button><button class="secondary" type="button" id="cancelSetup">Cancel</button><span class="spacer"></span><button class="primary" id="reviewSetup" ${selected && state.verifiedPath === selected ? '' : 'disabled'}>Continue to review</button></div>`
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
import { setTimeout as delay } from 'node:timers/promises'
|
|
2
|
+
import { isTransient } from './transport/reliable.js'
|
|
3
|
+
|
|
4
|
+
/** Balance by bytes, then let idle workers steal pending work from other lanes. */
|
|
5
|
+
function balance(jobs, count) {
|
|
6
|
+
const lanes = Array.from({ length: count }, () => ({ jobs: [], bytes: 0 }))
|
|
7
|
+
for (const job of [...jobs].sort((a, b) => b.item.bytes - a.item.bytes || a.order - b.order)) {
|
|
8
|
+
const lane = lanes.reduce((best, next) => next.bytes < best.bytes || next.bytes === best.bytes && next.jobs.length < best.jobs.length ? next : best)
|
|
9
|
+
lane.jobs.push(job)
|
|
10
|
+
lane.bytes += job.item.bytes
|
|
11
|
+
}
|
|
12
|
+
return lanes.map(lane => lane.jobs.reverse())
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
/** Each worker owns a connection. A failed connection never strands its files. */
|
|
16
|
+
export async function runUploadQueue(items, {
|
|
17
|
+
createWorker, transport, upload, succeeded = async () => {}, concurrency = 3,
|
|
18
|
+
retries = 2, retryDelayMs = 250, onProgress, onFileResult, onRetry, onWorkerState
|
|
19
|
+
}) {
|
|
20
|
+
if (!Number.isInteger(concurrency) || concurrency < 1 || concurrency > 8) throw new Error('Parallel uploads must be an integer from 1 to 8')
|
|
21
|
+
if (!Number.isInteger(retries) || retries < 0 || retries > 5) throw new Error('Upload retries must be an integer from 0 to 5')
|
|
22
|
+
const total = items.length
|
|
23
|
+
let pending = items.map((item, order) => ({ item, order, attempts: 0 }))
|
|
24
|
+
const done = new Map(), failed = new Map()
|
|
25
|
+
let bytes = 0
|
|
26
|
+
let lastError = new Error('No upload connection is available')
|
|
27
|
+
const finish = (job, error, workerId) => {
|
|
28
|
+
if (error) failed.set(job.order, { rel: job.item.rel, error: error.message, retryable: isTransient(error) })
|
|
29
|
+
else { done.set(job.order, { rel: job.item.rel, bytes: job.item.bytes }); bytes += job.item.bytes }
|
|
30
|
+
onFileResult?.({ item: job.item, index: done.size + failed.size - 1, total, workerId,
|
|
31
|
+
success: !error, ...(error ? { error: error.message } : {}), dryRun: false, bytes })
|
|
32
|
+
}
|
|
33
|
+
// Replacement rounds are bounded too: an outage cannot reconnect forever.
|
|
34
|
+
for (let round = 0; pending.length && round <= retries; round++) {
|
|
35
|
+
if (round) await delay(retryDelayMs * 2 ** (round - 1))
|
|
36
|
+
const count = Math.min(concurrency, pending.length)
|
|
37
|
+
const lanes = balance(pending, count)
|
|
38
|
+
const returned = []
|
|
39
|
+
let retryCursor = 0
|
|
40
|
+
let recoverable = false
|
|
41
|
+
const take = (id) => {
|
|
42
|
+
if (lanes[id].length) return lanes[id].pop()
|
|
43
|
+
const donor = lanes.reduce((best, lane) => lane.length > best.length ? lane : best, [])
|
|
44
|
+
if (donor.length) return donor.pop()
|
|
45
|
+
return retryCursor < returned.length ? returned[retryCursor++] : undefined
|
|
46
|
+
}
|
|
47
|
+
const requeue = (job, error, workerId, connectionFailure = false) => {
|
|
48
|
+
lastError = error
|
|
49
|
+
recoverable ||= isTransient(error)
|
|
50
|
+
job.error = error
|
|
51
|
+
returned.push(job)
|
|
52
|
+
onRetry?.({ item: job.item, error: error.message, workerId,
|
|
53
|
+
attempt: connectionFailure ? round + 1 : job.attempts, retries, connectionFailure })
|
|
54
|
+
}
|
|
55
|
+
const workers = await Promise.allSettled(lanes.map(async (_, id) => {
|
|
56
|
+
let client = round === 0 && id === 0 && transport && !transport.closed ? transport : null
|
|
57
|
+
const borrowed = !!client
|
|
58
|
+
let retired = false
|
|
59
|
+
try {
|
|
60
|
+
for (let job; (job = take(id));) {
|
|
61
|
+
if (!client) {
|
|
62
|
+
onWorkerState?.({ workerId: id + 1, state: 'connecting', file: job.item.rel })
|
|
63
|
+
try { client = await createWorker({ workerId: id + 1 }) }
|
|
64
|
+
catch (error) {
|
|
65
|
+
retired = true
|
|
66
|
+
onWorkerState?.({ workerId: id + 1, state: 'failed', file: job.item.rel })
|
|
67
|
+
requeue(job, error, id + 1, true)
|
|
68
|
+
return
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
onWorkerState?.({ workerId: id + 1, state: 'uploading', file: job.item.rel })
|
|
72
|
+
job.attempts++
|
|
73
|
+
onProgress?.({ phase: 'upload', item: job.item, index: done.size + failed.size, total, workerId: id + 1, workers: count, attempt: job.attempts })
|
|
74
|
+
try {
|
|
75
|
+
await upload(client, job.item)
|
|
76
|
+
} catch (error) {
|
|
77
|
+
if (isTransient(error)) {
|
|
78
|
+
retired = true
|
|
79
|
+
onWorkerState?.({ workerId: id + 1, state: 'failed', file: job.item.rel })
|
|
80
|
+
recoverable = true
|
|
81
|
+
lastError = error
|
|
82
|
+
if (job.attempts <= retries) requeue(job, error, id + 1)
|
|
83
|
+
else finish(job, error, id + 1)
|
|
84
|
+
return
|
|
85
|
+
}
|
|
86
|
+
finish(job, error, id + 1)
|
|
87
|
+
continue
|
|
88
|
+
}
|
|
89
|
+
try { await succeeded(job.item) }
|
|
90
|
+
catch (error) { finish(job, error, id + 1); continue }
|
|
91
|
+
finish(job, null, id + 1)
|
|
92
|
+
}
|
|
93
|
+
} finally {
|
|
94
|
+
if (client && (!borrowed || retired)) await client.close().catch(() => {})
|
|
95
|
+
if (!retired) onWorkerState?.({ workerId: id + 1, state: 'complete', file: null })
|
|
96
|
+
}
|
|
97
|
+
}))
|
|
98
|
+
const crashed = workers.find(worker => worker.status === 'rejected')
|
|
99
|
+
if (crashed) throw crashed.reason
|
|
100
|
+
pending = [...lanes.flat(), ...returned.slice(retryCursor)]
|
|
101
|
+
if (!recoverable) break
|
|
102
|
+
}
|
|
103
|
+
for (const job of pending) finish(job, job.error || lastError)
|
|
104
|
+
const ordered = map => [...map].sort(([a], [b]) => a - b).map(([, value]) => value)
|
|
105
|
+
return { done: ordered(done), failed: ordered(failed), bytes }
|
|
106
|
+
}
|
package/src/uploader.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import fs from 'node:fs'
|
|
2
|
+
import { runUploadQueue } from './uploadQueue.js'
|
|
2
3
|
import path from 'node:path'
|
|
3
4
|
import { validateRemotePath, relativeFile } from './paths.js'
|
|
4
5
|
import { hashFile } from './checksum.js'
|
|
@@ -59,7 +60,7 @@ export function collectFiles(projectRoot, matcher, { includeIgnored = false, onE
|
|
|
59
60
|
* Build the list of uploads without touching the network for writes.
|
|
60
61
|
* Saved projects compare local metadata with SQLite. Explicit scans verify remote content.
|
|
61
62
|
*/
|
|
62
|
-
export async function buildPlan(project, matcher, transport, { force = false,
|
|
63
|
+
export async function buildPlan(project, matcher, transport, { force = false, remoteInventory, onProgress } = {}) {
|
|
63
64
|
const { localPath, remotePath } = project
|
|
64
65
|
validateRemotePath(remotePath)
|
|
65
66
|
if (!fs.statSync(localPath).isDirectory()) throw new Error('Local source must be a directory')
|
|
@@ -70,7 +71,7 @@ export async function buildPlan(project, matcher, transport, { force = false, sc
|
|
|
70
71
|
if (excluded.length < 200) excluded.push(entry)
|
|
71
72
|
} })
|
|
72
73
|
const scope = project.syncScope
|
|
73
|
-
const saved = scope && !
|
|
74
|
+
const saved = scope && !remoteInventory ? (await import('./core/syncState.js')).readSyncFiles(scope) : null
|
|
74
75
|
const matches = []
|
|
75
76
|
const uploads = []
|
|
76
77
|
const skipped = []
|
|
@@ -98,7 +99,7 @@ export async function buildPlan(project, matcher, transport, { force = false, sc
|
|
|
98
99
|
continue
|
|
99
100
|
}
|
|
100
101
|
} else {
|
|
101
|
-
const remoteStat = await transport.stat(remote)
|
|
102
|
+
const remoteStat = remoteInventory ? remoteInventory.get(rel) : await transport.stat(remote)
|
|
102
103
|
if (remoteStat?.isDir) throw new Error(`Remote file destination is a directory: ${remote}`)
|
|
103
104
|
if (!force && remoteStat && remoteStat.size === localStat.size &&
|
|
104
105
|
await transport.checksum(remote) === await hashFile(abs)) {
|
|
@@ -125,7 +126,23 @@ export async function buildPlan(project, matcher, transport, { force = false, sc
|
|
|
125
126
|
/** Scan server content without uploading; only confirmed matches become the baseline. */
|
|
126
127
|
export async function scanServer(project, matcher, transport, options = {}) {
|
|
127
128
|
if (!project.syncScope) throw new Error('A saved project is required for a server scan')
|
|
128
|
-
const
|
|
129
|
+
const root = validateRemotePath(project.remotePath)
|
|
130
|
+
const remoteInventory = new Map()
|
|
131
|
+
const folders = ['']
|
|
132
|
+
for (let index = 0; index < folders.length; index++) {
|
|
133
|
+
const folder = folders[index]
|
|
134
|
+
const remote = path.posix.join(root, folder)
|
|
135
|
+
options.onProgress?.({ phase: 'inventory', item: { rel: remote }, index, total: folders.length })
|
|
136
|
+
for (const entry of await transport.list(remote)) {
|
|
137
|
+
if (!entry.name || entry.name === '.' || entry.name === '..' || /[\\/\x00-\x1f\x7f]/.test(entry.name)) continue
|
|
138
|
+
const rel = folder ? folder + '/' + entry.name : entry.name
|
|
139
|
+
const isDir = entry.type === 'dir'
|
|
140
|
+
if (entry.type !== 'file' && !isDir) continue
|
|
141
|
+
remoteInventory.set(rel, { isDir, size: entry.size })
|
|
142
|
+
if (isDir && matcher.shouldWatch(path.join(project.localPath, ...rel.split('/')))) folders.push(rel)
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
const plan = await buildPlan(project, matcher, transport, { ...options, force: false, remoteInventory })
|
|
129
146
|
const { replaceScannedFiles } = await import('./core/syncState.js')
|
|
130
147
|
replaceScannedFiles(project.syncScope, plan.matches)
|
|
131
148
|
return plan
|
|
@@ -135,8 +152,8 @@ export async function scanServer(project, matcher, transport, options = {}) {
|
|
|
135
152
|
* Upload one file atomically: write to <name>.devsync-uploading, then rename.
|
|
136
153
|
* Without this, PHP can parse a half-written file and 500 mid-transfer.
|
|
137
154
|
*/
|
|
138
|
-
export async function uploadOne(transport, item) {
|
|
139
|
-
if (transport.transaction) return transport.transaction((raw) => uploadAttempt(raw, item))
|
|
155
|
+
export async function uploadOne(transport, item, options) {
|
|
156
|
+
if (transport.transaction) return transport.transaction((raw) => uploadAttempt(raw, item), options)
|
|
140
157
|
return uploadAttempt(transport, item)
|
|
141
158
|
}
|
|
142
159
|
|
|
@@ -155,7 +172,19 @@ async function uploadAttempt(transport, item) {
|
|
|
155
172
|
}
|
|
156
173
|
}
|
|
157
174
|
|
|
158
|
-
export async function runPlan(plan, transport, { dryRun = false, onProgress, onFileResult } = {}) {
|
|
175
|
+
export async function runPlan(plan, transport, { dryRun = false, onProgress, onFileResult, createWorker, concurrency = 3, retries = 2, retryDelayMs = 250, onRetry, onWorkerState } = {}) {
|
|
176
|
+
if (!dryRun && createWorker) {
|
|
177
|
+
return runUploadQueue(plan.uploads, {
|
|
178
|
+
transport, createWorker, concurrency, retries, retryDelayMs, onProgress, onFileResult, onRetry, onWorkerState,
|
|
179
|
+
upload: (client, item) => uploadOne(client, item, { retries: 0 }),
|
|
180
|
+
succeeded: async item => {
|
|
181
|
+
if (plan.syncScope) {
|
|
182
|
+
const { recordUploadedFile } = await import('./core/syncState.js')
|
|
183
|
+
recordUploadedFile(plan.syncScope, item)
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
})
|
|
187
|
+
}
|
|
159
188
|
const done = []
|
|
160
189
|
const failed = []
|
|
161
190
|
let bytes = 0
|
package/src/watcher.js
CHANGED
|
@@ -10,6 +10,9 @@ export class ProjectWatcher {
|
|
|
10
10
|
this.matcher = matcher
|
|
11
11
|
this.transport = transport
|
|
12
12
|
this.debounceMs = opts.debounceMs ?? 400
|
|
13
|
+
this.createWorker = opts.createWorker
|
|
14
|
+
this.concurrency = opts.concurrency ?? 3
|
|
15
|
+
this.onWorkerState = opts.onWorkerState
|
|
13
16
|
this.dryRun = opts.dryRun ?? false
|
|
14
17
|
this.onEvent = opts.onEvent ?? (() => {})
|
|
15
18
|
this.onSettle = opts.onSettle ?? (() => {})
|
|
@@ -84,7 +87,9 @@ export class ProjectWatcher {
|
|
|
84
87
|
const plan = await buildPlan(this.project, this.matcher, this.transport, { onProgress: this.onProgress })
|
|
85
88
|
this.onPlan(plan)
|
|
86
89
|
const result = await runPlan(plan, this.transport, {
|
|
87
|
-
dryRun: this.dryRun,
|
|
90
|
+
dryRun: this.dryRun, createWorker: this.createWorker, concurrency: this.concurrency,
|
|
91
|
+
onProgress: this.onProgress, onFileResult: this.onFileResult, onWorkerState: this.onWorkerState,
|
|
92
|
+
onRetry: event => this.onEvent({ type: 'retry', message: `Queue ${event.workerId}: ${event.connectionFailure ? 'connection unavailable' : event.item.rel + ' interrupted'}; returned to the pending queue.` })
|
|
88
93
|
})
|
|
89
94
|
this.stats.cycles++
|
|
90
95
|
if (!this.dryRun) this.stats.uploaded += result.done.length
|
|
@@ -119,4 +124,4 @@ export class ProjectWatcher {
|
|
|
119
124
|
if (this.watcher) await this.watcher.close()
|
|
120
125
|
await this.cycle
|
|
121
126
|
}
|
|
122
|
-
}
|
|
127
|
+
}
|