shared-hosting-sync 0.1.4 → 0.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +3 -0
- package/CHANGELOG.md +19 -0
- package/README.md +42 -8
- package/RELEASING.md +7 -7
- package/package.json +2 -2
- package/src/arguments.js +1 -1
- package/src/cli.js +9 -5
- package/src/core/deployment.js +1 -0
- package/src/core/envSchema.js +9 -0
- package/src/core/syncState.js +12 -4
- package/src/server/deployments.js +18 -4
- package/src/transport/ftp.js +7 -11
- package/src/transport/reliable.js +3 -3
- package/src/transport/sftp.js +12 -3
- package/src/ui/deployments.js +4 -2
- package/src/ui/style.css +2 -0
- package/src/ui/ui.js +1 -1
- package/src/uploadQueue.js +106 -0
- package/src/uploader.js +51 -23
- package/src/watcher.js +7 -2
package/.env.example
CHANGED
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,24 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.1.6 — 18 November 2026
|
|
4
|
+
|
|
5
|
+
- Reuse destination folder setup across each upload connection.
|
|
6
|
+
- Avoid an extra server request for every successful file by using unique temporary
|
|
7
|
+
names and renaming only after transfer.
|
|
8
|
+
- Reduce repeated local path and SQLite statement preparation during large syncs.
|
|
9
|
+
|
|
10
|
+
## 0.1.5 — 18 November 2026
|
|
11
|
+
|
|
12
|
+
- List server folders before comparing local files during an initial scan, avoiding
|
|
13
|
+
a separate server metadata request for every local file. Verified matches are
|
|
14
|
+
saved to SQLite before the next sync.
|
|
15
|
+
- Upload through three independent connections by default, balanced by file size.
|
|
16
|
+
Idle workers take pending work from other queues.
|
|
17
|
+
- Return interrupted files to the pending queue for healthy workers, with bounded
|
|
18
|
+
retries and completion-based progress. Failed files remain pending in SQLite.
|
|
19
|
+
- Show individual queue status in the dashboard and allow 1–8 parallel uploads
|
|
20
|
+
through project settings, DEVSYNC_CONCURRENCY, or --concurrency.
|
|
21
|
+
|
|
3
22
|
## 0.1.4 — 27 October 2026
|
|
4
23
|
|
|
5
24
|
- Use persistent per-file sync records for fast local size and timestamp checks.
|
package/README.md
CHANGED
|
@@ -6,8 +6,7 @@ choose a local folder and a server destination, then deploy or keep files in syn
|
|
|
6
6
|
Use the terminal or a local browser dashboard. Both interfaces control the same
|
|
7
7
|
projects, settings, and deployment operations.
|
|
8
8
|
|
|
9
|
-
>
|
|
10
|
-
> CLI and UI onboarding, previews, deployment, sync, and watching are implemented.
|
|
9
|
+
> Requires Node.js 24 or newer.
|
|
11
10
|
|
|
12
11
|
The package name is **shared-hosting-sync**. Run it as `shared-hosting-sync` or
|
|
13
12
|
use the shorter `devsync` alias shown in examples below. Both commands run the
|
|
@@ -44,10 +43,11 @@ shared-hosting-sync --version
|
|
|
44
43
|
shared-hosting-sync ui
|
|
45
44
|
```
|
|
46
45
|
|
|
47
|
-
Alternatively,
|
|
46
|
+
Alternatively, run `npm run release:check` from the checkout and install the resulting
|
|
47
|
+
archive. Replace `<version>` with its actual version:
|
|
48
48
|
|
|
49
49
|
```sh
|
|
50
|
-
npm install -g ./artifacts/shared-hosting-sync
|
|
50
|
+
npm install -g "./artifacts/shared-hosting-sync-<version>.tgz"
|
|
51
51
|
```
|
|
52
52
|
|
|
53
53
|
For development, install from this checkout:
|
|
@@ -67,12 +67,46 @@ name conflict:
|
|
|
67
67
|
|
|
68
68
|
```sh
|
|
69
69
|
npm uninstall -g devsync
|
|
70
|
-
npm install -g
|
|
70
|
+
npm install -g shared-hosting-sync
|
|
71
71
|
```
|
|
72
72
|
|
|
73
73
|
This replaces the old global installation. Saved projects and credentials remain
|
|
74
|
-
in `~/.devsync/` (or your configured `DEVSYNC_HOME`).
|
|
75
|
-
|
|
74
|
+
in `~/.devsync/` (or your configured `DEVSYNC_HOME`). To back up the settings,
|
|
75
|
+
stop DevSync and copy that directory somewhere private. Keep the database and
|
|
76
|
+
`master.key` together so saved credentials can be restored.
|
|
77
|
+
|
|
78
|
+
## First sync for an existing site
|
|
79
|
+
|
|
80
|
+
1. Choose **Scan server / Initial sync** and confirm. DevSync lists the destination
|
|
81
|
+
folders first, then checks local files against that inventory. Files with the
|
|
82
|
+
same size are checked by content before being marked as synced.
|
|
83
|
+
2. Review pending files, then choose **Sync changes** to upload them. The scan
|
|
84
|
+
saves verified matches to SQLite and does not upload or delete server files.
|
|
85
|
+
3. For everyday updates, use **Sync changes** directly. It uses saved records and
|
|
86
|
+
only queues new or changed files. For an empty destination, skip the scan and
|
|
87
|
+
use **Sync changes** or **Deploy all files**.
|
|
88
|
+
|
|
89
|
+
## Parallel uploads
|
|
90
|
+
|
|
91
|
+
Deploy, Sync, and Watch use up to **3 upload queues** by default. Each queue owns
|
|
92
|
+
an independent connection to the same host and configured port, so FTP commands
|
|
93
|
+
never overlap on one control connection. FTP/FTPS data connections still use the
|
|
94
|
+
server's passive ports. SFTP uses separate SSH/SFTP sessions.
|
|
95
|
+
|
|
96
|
+
Files are initially balanced by byte size. Idle queues take pending work from
|
|
97
|
+
other queues. If a connection drops, its interrupted file returns to the pending
|
|
98
|
+
list and healthy queues continue. A file gets at most two retries after its first
|
|
99
|
+
attempt; replacement connection rounds are also bounded. Permission errors are
|
|
100
|
+
reported immediately. Failed files remain pending for a later sync. An uncertain
|
|
101
|
+
rename retries the entire temporary-file upload.
|
|
102
|
+
|
|
103
|
+
Set **Parallel uploads** under Edit setup → Hosting → SSH key and advanced
|
|
104
|
+
settings, or use `DEVSYNC_CONCURRENCY=3` in the project's environment file.
|
|
105
|
+
The CLI also accepts `devsync sync [label] --concurrency 3`. Values from 1 to 8
|
|
106
|
+
are supported; choose 1 for restrictive hosting accounts. Small batches open only
|
|
107
|
+
as many connections as needed, and previews or unchanged syncs open none.
|
|
108
|
+
Stopping Watch finishes the current upload batch, including bounded retries,
|
|
109
|
+
then closes its connections. Progress counts completed files, never retry attempts.
|
|
76
110
|
|
|
77
111
|
## Start from a project folder
|
|
78
112
|
|
|
@@ -350,7 +384,7 @@ Provision any server-side application secrets separately.
|
|
|
350
384
|
- Records belong to a source folder and destination (protocol, host, port, account,
|
|
351
385
|
and server folder). Changing either starts a separate baseline.
|
|
352
386
|
- If files already exist on the server, use **Scan server / Initial sync** in the
|
|
353
|
-
dashboard once before your first sync. It asks for confirmation, compares server
|
|
387
|
+
dashboard once before your first sync. It asks for confirmation, lists server folders first, then compares local and server
|
|
354
388
|
content using SHA-256 for equal-size files, and saves only matching files. It
|
|
355
389
|
uploads or deletes nothing; use Sync changes afterward for pending files.
|
|
356
390
|
Terminal equivalent: `devsync scan [label]` (or `--confirm` for automation).
|
package/RELEASING.md
CHANGED
|
@@ -24,7 +24,7 @@ aliases, SQLite storage, environment onboarding, real FTP deploy/sync, dashboard
|
|
|
24
24
|
startup, API identity, and UI assets must pass. Nothing is installed into the
|
|
25
25
|
developer's usual global prefix. Generated files are under `artifacts/`:
|
|
26
26
|
|
|
27
|
-
- `shared-hosting-sync
|
|
27
|
+
- `shared-hosting-sync-<version>.tgz` (version follows package.json)
|
|
28
28
|
- `release.json`, including the complete package file list
|
|
29
29
|
- `SHA256SUMS`
|
|
30
30
|
|
|
@@ -45,9 +45,9 @@ npm whoami
|
|
|
45
45
|
npm view shared-hosting-sync name version
|
|
46
46
|
```
|
|
47
47
|
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
the
|
|
48
|
+
Confirm the signed-in account is the intended maintainer and that the candidate
|
|
49
|
+
version has not already been published. Replace `<version>` in the examples with
|
|
50
|
+
the candidate version from package.json. Use npm's browser sign-in and
|
|
51
51
|
complete any account/2FA requirements personally; do not put tokens into source
|
|
52
52
|
files or chat.
|
|
53
53
|
|
|
@@ -55,7 +55,7 @@ Once the account, version, passing CI, package manifest, and checksum are review
|
|
|
55
55
|
publish the exact checked archive:
|
|
56
56
|
|
|
57
57
|
```sh
|
|
58
|
-
npm publish ./artifacts/shared-hosting-sync
|
|
58
|
+
npm publish "./artifacts/shared-hosting-sync-<version>.tgz" --access public
|
|
59
59
|
```
|
|
60
60
|
|
|
61
61
|
Publishing with `--access public` makes the package contents publicly available. Publishing
|
|
@@ -66,8 +66,8 @@ published version; fix issues in a new version.
|
|
|
66
66
|
## Verify publication
|
|
67
67
|
|
|
68
68
|
```sh
|
|
69
|
-
npm view shared-hosting-sync
|
|
70
|
-
npm install -g shared-hosting-sync
|
|
69
|
+
npm view "shared-hosting-sync@<version>" name version dist.integrity
|
|
70
|
+
npm install -g "shared-hosting-sync@<version>"
|
|
71
71
|
shared-hosting-sync --version
|
|
72
72
|
devsync --version
|
|
73
73
|
shared-hosting-sync ui
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "shared-hosting-sync",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.6",
|
|
4
4
|
"description": "Deploy and sync local projects to shared hosting over FTP, FTPS, or SFTP, with a CLI and local dashboard.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
|
@@ -49,7 +49,7 @@
|
|
|
49
49
|
"scripts": {
|
|
50
50
|
"start": "node src/cli.js",
|
|
51
51
|
"release:check": "node scripts/release-check.js",
|
|
52
|
-
"test": "node test/ignore.test.js && node --test --test-timeout=60000 --test-concurrency=1 test/uploader.test.js test/sync-state.test.js test/transport-ftp.test.js test/cli-deployment.test.js test/cli-onboarding.test.js test/onboarding.test.js test/ui-onboarding.test.js test/ui-deployments.test.js test/ui-sync.test.js test/reliability.test.js test/protocols.test.js test/transport-sftp.test.js test/watcher.test.js test/envResolver.test.js test/db.test.js test/secrets.test.js",
|
|
52
|
+
"test": "node test/ignore.test.js && node --test --test-timeout=60000 --test-concurrency=1 test/uploader.test.js test/upload-queue.test.js test/sync-state.test.js test/transport-ftp.test.js test/cli-deployment.test.js test/cli-onboarding.test.js test/onboarding.test.js test/ui-onboarding.test.js test/ui-deployments.test.js test/ui-sync.test.js test/reliability.test.js test/protocols.test.js test/transport-sftp.test.js test/watcher.test.js test/envResolver.test.js test/db.test.js test/secrets.test.js",
|
|
53
53
|
"prepack": "npm test",
|
|
54
54
|
"prepublishOnly": "npm test"
|
|
55
55
|
},
|
package/src/arguments.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
const VALUE_FLAGS = new Set(['transport', 'env-file', 'remote-path', 'host', 'username', 'port', 'label', 'path'])
|
|
1
|
+
const VALUE_FLAGS = new Set(['concurrency', 'transport', 'env-file', 'remote-path', 'host', 'username', 'port', 'label', 'path'])
|
|
2
2
|
const BOOL_FLAGS = new Set(['confirm', 'dry-run', 'force', 'plan', 'sync', 'edit', 'browse', 'browser', 'help', 'version'])
|
|
3
3
|
|
|
4
4
|
export function parseArgs(argv) {
|
package/src/cli.js
CHANGED
|
@@ -64,7 +64,7 @@ try {
|
|
|
64
64
|
const sessionOverrides = new Map()
|
|
65
65
|
const prompt = createPrompter()
|
|
66
66
|
const flagOverrides = () => Object.fromEntries([
|
|
67
|
-
['transport', 'DEVSYNC_TRANSPORT'], ['host', 'DEVSYNC_HOST'],
|
|
67
|
+
['concurrency', 'DEVSYNC_CONCURRENCY'], ['transport', 'DEVSYNC_TRANSPORT'], ['host', 'DEVSYNC_HOST'],
|
|
68
68
|
['username', 'DEVSYNC_USERNAME'], ['port', 'DEVSYNC_PORT'], ['remote-path', 'DEVSYNC_REMOTE_PATH']
|
|
69
69
|
].filter(([flag]) => flags[flag] !== undefined).map(([flag, key]) => [key, flags[flag]]))
|
|
70
70
|
|
|
@@ -137,7 +137,7 @@ async function credsForProject(project, { deployment = false } = {}) {
|
|
|
137
137
|
envFile: flags['env-file'], overrides: { ...flagOverrides(), ...sessionOverrides.get(project.id) }
|
|
138
138
|
})
|
|
139
139
|
Object.assign(project, context.project)
|
|
140
|
-
return { ...context.creds, dryRun: context.dryRun, debounceMs: context.debounceMs }
|
|
140
|
+
return { ...context.creds, dryRun: context.dryRun, debounceMs: context.debounceMs, concurrency: context.concurrency }
|
|
141
141
|
}
|
|
142
142
|
const r = resolveEnv(project.id, {
|
|
143
143
|
envFile: flags['env-file'], overrides: { ...flagOverrides(), ...sessionOverrides.get(project.id) }
|
|
@@ -331,11 +331,12 @@ const COMMANDS = {
|
|
|
331
331
|
setStatus(proj.id, 'idle')
|
|
332
332
|
return
|
|
333
333
|
}
|
|
334
|
-
t = await createTransport(creds, undefined, { onRetry: retryNotice })
|
|
335
334
|
let lastPct = -1
|
|
336
335
|
const r = await runPlan(p, t, {
|
|
337
|
-
dryRun,
|
|
338
|
-
|
|
336
|
+
dryRun, concurrency: creds.concurrency,
|
|
337
|
+
createWorker: () => createTransport(creds, undefined, { retries: 0 }),
|
|
338
|
+
onRetry: event => warn(`Queue ${event.workerId}: ${event.connectionFailure ? 'connection unavailable' : event.item.rel + ' interrupted'}; returned to the pending queue.`),
|
|
339
|
+
onFileResult: ({ index, total, item }) => {
|
|
339
340
|
const pct = Math.floor(((index + 1) / total) * 100)
|
|
340
341
|
if (pct !== lastPct && pct % 10 === 0) {
|
|
341
342
|
lastPct = pct
|
|
@@ -400,6 +401,8 @@ const COMMANDS = {
|
|
|
400
401
|
const w = new ProjectWatcher(proj, matcher, t, {
|
|
401
402
|
dryRun: creds.dryRun || !!flags['dry-run'],
|
|
402
403
|
debounceMs: creds.debounceMs,
|
|
404
|
+
concurrency: creds.concurrency,
|
|
405
|
+
createWorker: () => createTransport(creds, flags.transport, { retries: 0 }),
|
|
403
406
|
onEvent: (e) => {
|
|
404
407
|
if (e.type === 'error') {
|
|
405
408
|
setStatus(proj.id, 'error', { error: e.message })
|
|
@@ -503,6 +506,7 @@ ${C.b('options')}
|
|
|
503
506
|
--edit setup: edit connection settings
|
|
504
507
|
--browse setup: choose a different server folder
|
|
505
508
|
--label <name> name a new project
|
|
509
|
+
--concurrency <1-8> parallel upload queues (default: 3)
|
|
506
510
|
--confirm confirm a first-time server scan
|
|
507
511
|
--dry-run --force preview / include unchanged files
|
|
508
512
|
|
package/src/core/deployment.js
CHANGED
|
@@ -18,6 +18,7 @@ export function resolveDeployment(project, options = {}) {
|
|
|
18
18
|
matcher: buildMatcher(target.localPath),
|
|
19
19
|
creds: credsFromResolved(resolved.values),
|
|
20
20
|
dryRun: ['true', '1', true].includes(resolved.values.DEVSYNC_DRY_RUN),
|
|
21
|
+
concurrency: Number(resolved.values.DEVSYNC_CONCURRENCY),
|
|
21
22
|
debounceMs: Number(resolved.values.DEVSYNC_DEBOUNCE_MS),
|
|
22
23
|
values: resolved.values
|
|
23
24
|
}
|
package/src/core/envSchema.js
CHANGED
|
@@ -82,6 +82,15 @@ export const ENV_SCHEMA = [
|
|
|
82
82
|
label: 'Dry run',
|
|
83
83
|
help: 'Plan and report without uploading.'
|
|
84
84
|
},
|
|
85
|
+
{
|
|
86
|
+
key: 'DEVSYNC_CONCURRENCY',
|
|
87
|
+
type: 'int',
|
|
88
|
+
default: 3,
|
|
89
|
+
min: 1,
|
|
90
|
+
max: 8,
|
|
91
|
+
label: 'Parallel uploads',
|
|
92
|
+
help: 'Upload queues (1–8). Default 3. Use 1 if your host limits connections.'
|
|
93
|
+
},
|
|
85
94
|
{
|
|
86
95
|
key: 'DEVSYNC_DEBOUNCE_MS',
|
|
87
96
|
type: 'int',
|
package/src/core/syncState.js
CHANGED
|
@@ -27,11 +27,19 @@ export function syncStatus(scope) {
|
|
|
27
27
|
return { trackedFiles: count, scannedAt: target?.scanned_at || null, ready: !!target }
|
|
28
28
|
}
|
|
29
29
|
|
|
30
|
+
const statements = new WeakMap()
|
|
31
|
+
function prepared(db, sql) {
|
|
32
|
+
let cache = statements.get(db)
|
|
33
|
+
if (!cache) statements.set(db, cache = new Map())
|
|
34
|
+
if (!cache.has(sql)) cache.set(sql, db.prepare(sql))
|
|
35
|
+
return cache.get(sql)
|
|
36
|
+
}
|
|
37
|
+
|
|
30
38
|
function ensureTarget(db, scope) {
|
|
31
|
-
db
|
|
39
|
+
prepared(db, 'INSERT OR IGNORE INTO sync_targets (project_id, target_key) VALUES (?, ?)').run(scope.projectId, scope.targetKey)
|
|
32
40
|
}
|
|
33
41
|
function writeFile(db, scope, rel, value) {
|
|
34
|
-
db
|
|
42
|
+
prepared(db, `INSERT INTO sync_files (project_id, target_key, relative_path, size, mtime_ms, ctime_ms, synced_at)
|
|
35
43
|
VALUES (?, ?, ?, ?, ?, ?, ?) ON CONFLICT(project_id, target_key, relative_path) DO UPDATE SET
|
|
36
44
|
size=excluded.size, mtime_ms=excluded.mtime_ms, ctime_ms=excluded.ctime_ms, synced_at=excluded.synced_at`)
|
|
37
45
|
.run(scope.projectId, scope.targetKey, rel, value.size, value.mtimeMs, value.ctimeMs, new Date().toISOString())
|
|
@@ -41,7 +49,7 @@ function writeFile(db, scope, rel, value) {
|
|
|
41
49
|
export function recordUploadedFile(scope, item) {
|
|
42
50
|
const db = openDb()
|
|
43
51
|
if (!sameFile(item.fingerprint, currentFingerprint(item.abs))) {
|
|
44
|
-
db
|
|
52
|
+
prepared(db, 'DELETE FROM sync_files WHERE project_id = ? AND target_key = ? AND relative_path = ?').run(scope.projectId, scope.targetKey, item.rel)
|
|
45
53
|
return
|
|
46
54
|
}
|
|
47
55
|
ensureTarget(db, scope)
|
|
@@ -54,7 +62,7 @@ export function replaceScannedFiles(scope, matches) {
|
|
|
54
62
|
db.exec('BEGIN IMMEDIATE')
|
|
55
63
|
try {
|
|
56
64
|
ensureTarget(db, scope)
|
|
57
|
-
db
|
|
65
|
+
prepared(db, 'DELETE FROM sync_files WHERE project_id = ? AND target_key = ?').run(scope.projectId, scope.targetKey)
|
|
58
66
|
for (const item of matches) {
|
|
59
67
|
if (sameFile(item.fingerprint, currentFingerprint(item.abs))) writeFile(db, scope, item.rel, item.fingerprint)
|
|
60
68
|
}
|
|
@@ -60,7 +60,7 @@ export class DeploymentService {
|
|
|
60
60
|
}
|
|
61
61
|
|
|
62
62
|
progress(record, event) {
|
|
63
|
-
if (!record.stopRequested) record.view.phase = event.phase === 'planning' ? 'planning' : 'uploading'
|
|
63
|
+
if (!record.stopRequested) record.view.phase = event.phase === 'inventory' ? 'inventory' : event.phase === 'planning' ? 'planning' : 'uploading'
|
|
64
64
|
record.view.progress = {
|
|
65
65
|
current: event.index, total: event.total, file: this.clean(record, event.item.rel),
|
|
66
66
|
bytes: record.view.progress?.bytes || 0
|
|
@@ -69,6 +69,7 @@ export class DeploymentService {
|
|
|
69
69
|
}
|
|
70
70
|
|
|
71
71
|
plan(record, plan) {
|
|
72
|
+
record.view.workers = []
|
|
72
73
|
record.view.plan = {
|
|
73
74
|
considered: plan.totalFiles, uploadCount: plan.uploads.length, skippedCount: plan.skipped.length,
|
|
74
75
|
bytes: plan.comparedBytes,
|
|
@@ -82,6 +83,15 @@ export class DeploymentService {
|
|
|
82
83
|
this.log(record, `Compared ${plan.totalFiles} files: ${plan.uploads.length} to upload, ${plan.skipped.length} unchanged`)
|
|
83
84
|
}
|
|
84
85
|
|
|
86
|
+
workerState(record, event) {
|
|
87
|
+
record.view.workers ||= []
|
|
88
|
+
const current = record.view.workers.findIndex(worker => worker.workerId === event.workerId)
|
|
89
|
+
const value = { workerId: event.workerId, state: event.state, file: event.file ? this.clean(record, event.file) : null }
|
|
90
|
+
if (current < 0) record.view.workers.push(value)
|
|
91
|
+
else record.view.workers[current] = value
|
|
92
|
+
record.view.updatedAt = stamp()
|
|
93
|
+
}
|
|
94
|
+
|
|
85
95
|
fileResult(record, event) {
|
|
86
96
|
record.view.progress = { current: event.index + 1, total: event.total, file: this.clean(record, event.item.rel), bytes: event.bytes }
|
|
87
97
|
this.log(record, event.success ? `${event.dryRun ? 'Would upload' : 'Uploaded'} ${event.item.rel}` : `Failed ${event.item.rel}: ${event.error}`, event.success ? 'info' : 'error')
|
|
@@ -125,7 +135,7 @@ export class DeploymentService {
|
|
|
125
135
|
projectId: project.id, operationId: crypto.randomUUID(), kind,
|
|
126
136
|
mode: kind === 'preview' ? options.mode || 'sync' : kind === 'deploy' ? 'deploy' : 'sync',
|
|
127
137
|
state: 'running', phase: 'connecting', dryRun, plan: null, result: null, progress: null,
|
|
128
|
-
logs: [], lastError: null, startedAt: stamp(), updatedAt: stamp(), remotePath: context.project.remotePath
|
|
138
|
+
logs: [], workers: [], concurrency: context.concurrency, lastError: null, startedAt: stamp(), updatedAt: stamp(), remotePath: context.project.remotePath
|
|
129
139
|
}
|
|
130
140
|
}
|
|
131
141
|
this.records.set(project.id, record)
|
|
@@ -151,6 +161,7 @@ export class DeploymentService {
|
|
|
151
161
|
}
|
|
152
162
|
if (record.stopRequested || this.closing) return
|
|
153
163
|
const context = record.context
|
|
164
|
+
const createWorker = () => this.connect(context.creds, undefined, { retries: 0 })
|
|
154
165
|
if (record.view.kind === 'scan') {
|
|
155
166
|
await connect()
|
|
156
167
|
const plan = await scanServer(context.project, context.matcher, record.transport, { onProgress: (event) => this.progress(record, event) })
|
|
@@ -168,6 +179,8 @@ export class DeploymentService {
|
|
|
168
179
|
if (record.stopRequested || this.closing) return
|
|
169
180
|
const watcher = this.watcherFactory(context.project, context.matcher, record.transport, {
|
|
170
181
|
dryRun: record.view.dryRun, debounceMs: context.debounceMs,
|
|
182
|
+
createWorker, concurrency: context.concurrency,
|
|
183
|
+
onWorkerState: event => this.workerState(record, event),
|
|
171
184
|
onProgress: (event) => this.progress(record, event),
|
|
172
185
|
onPlan: (plan) => this.plan(record, plan),
|
|
173
186
|
onFileResult: (event) => this.fileResult(record, event),
|
|
@@ -193,9 +206,10 @@ export class DeploymentService {
|
|
|
193
206
|
force: record.view.mode === 'deploy', onProgress: (event) => this.progress(record, event)
|
|
194
207
|
})
|
|
195
208
|
this.plan(record, plan)
|
|
196
|
-
if (!record.view.dryRun && plan.uploads.length) await connect()
|
|
197
209
|
const result = await runPlan(plan, record.transport, {
|
|
198
|
-
dryRun: record.view.dryRun,
|
|
210
|
+
dryRun: record.view.dryRun, createWorker, concurrency: context.concurrency,
|
|
211
|
+
onWorkerState: event => this.workerState(record, event),
|
|
212
|
+
onRetry: event => this.log(record, `Queue ${event.workerId}: ${event.connectionFailure ? 'connection unavailable' : event.item.rel + ' interrupted'}; returned to the pending queue.`, 'warning'),
|
|
199
213
|
onProgress: (event) => this.progress(record, event), onFileResult: (event) => this.fileResult(record, event)
|
|
200
214
|
})
|
|
201
215
|
this.complete(record, { done: result.done.length, failed: result.failed, bytes: result.bytes, dryRun: record.view.dryRun })
|
package/src/transport/ftp.js
CHANGED
|
@@ -9,6 +9,7 @@ import { hashSink } from '../checksum.js'
|
|
|
9
9
|
export class FtpTransport {
|
|
10
10
|
constructor(creds) {
|
|
11
11
|
this.creds = creds
|
|
12
|
+
this.preparedDirs = new Set()
|
|
12
13
|
this.client = null
|
|
13
14
|
}
|
|
14
15
|
|
|
@@ -27,6 +28,7 @@ export class FtpTransport {
|
|
|
27
28
|
c.close()
|
|
28
29
|
throw err
|
|
29
30
|
}
|
|
31
|
+
this.preparedDirs.clear()
|
|
30
32
|
this.client = c
|
|
31
33
|
return this
|
|
32
34
|
}
|
|
@@ -35,23 +37,17 @@ export class FtpTransport {
|
|
|
35
37
|
const items = await this.client.list(remotePath || '/')
|
|
36
38
|
return items.map((e) => ({
|
|
37
39
|
name: e.name,
|
|
38
|
-
type: e.isDirectory ? 'dir' : 'file',
|
|
40
|
+
type: e.isDirectory ? 'dir' : e.isFile ? 'file' : 'other',
|
|
39
41
|
size: e.size ?? 0,
|
|
40
42
|
mtime: e.modifyTime ? new Date(e.modifyTime) : null
|
|
41
43
|
}))
|
|
42
44
|
}
|
|
43
45
|
|
|
44
46
|
async ensureDir(remotePath) {
|
|
45
|
-
if (!remotePath) return
|
|
46
|
-
//
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
let acc = remotePath.startsWith('/') ? '' : '.'
|
|
50
|
-
for (const part of parts) {
|
|
51
|
-
acc = acc === '' || acc === '.' ? `${acc}/${part}` : `${acc}/${part}`
|
|
52
|
-
const current = acc.startsWith('/') ? acc : `/${acc}`
|
|
53
|
-
await this.client.ensureDir(current)
|
|
54
|
-
}
|
|
47
|
+
if (!remotePath || this.preparedDirs.has(remotePath)) return
|
|
48
|
+
// basic-ftp creates intermediate directories itself. Uploads use absolute paths.
|
|
49
|
+
await this.client.ensureDir(remotePath)
|
|
50
|
+
this.preparedDirs.add(remotePath)
|
|
55
51
|
}
|
|
56
52
|
|
|
57
53
|
async exists(remotePath) {
|
|
@@ -19,15 +19,15 @@ export class ReliableTransport {
|
|
|
19
19
|
|
|
20
20
|
async connect() { await this.transaction(async () => {}); return this }
|
|
21
21
|
|
|
22
|
-
async transaction(action) {
|
|
22
|
+
async transaction(action, { retries = this.retries } = {}) {
|
|
23
23
|
for (let attempt = 0; ; attempt++) {
|
|
24
24
|
if (this.closed) throw new Error('Transport has been closed')
|
|
25
25
|
try {
|
|
26
26
|
this.raw ||= await this.connectRaw()
|
|
27
27
|
return await action(this.raw)
|
|
28
28
|
} catch (error) {
|
|
29
|
-
if (!isTransient(error) || attempt >=
|
|
30
|
-
this.onRetry({ attempt: attempt + 1, retries
|
|
29
|
+
if (!isTransient(error) || attempt >= retries || this.closed) throw error
|
|
30
|
+
this.onRetry({ attempt: attempt + 1, retries })
|
|
31
31
|
if (this.raw) await this.raw.close().catch(() => {})
|
|
32
32
|
this.raw = null
|
|
33
33
|
await delay(this.retryDelayMs * 2 ** attempt)
|
package/src/transport/sftp.js
CHANGED
|
@@ -7,6 +7,7 @@ import { finished } from 'node:stream/promises'
|
|
|
7
7
|
export class SftpTransport {
|
|
8
8
|
constructor(creds) {
|
|
9
9
|
this.creds = creds
|
|
10
|
+
this.preparedDirs = new Set()
|
|
10
11
|
this.client = null
|
|
11
12
|
}
|
|
12
13
|
|
|
@@ -30,6 +31,7 @@ export class SftpTransport {
|
|
|
30
31
|
await c.end().catch(() => {})
|
|
31
32
|
throw err
|
|
32
33
|
}
|
|
34
|
+
this.preparedDirs.clear()
|
|
33
35
|
this.client = c
|
|
34
36
|
return this
|
|
35
37
|
}
|
|
@@ -40,17 +42,24 @@ export class SftpTransport {
|
|
|
40
42
|
.filter((e) => e.name !== '.' && e.name !== '..')
|
|
41
43
|
.map((e) => ({
|
|
42
44
|
name: e.name,
|
|
43
|
-
type: e.type === 'd' ? 'dir' : 'file',
|
|
45
|
+
type: e.type === 'd' ? 'dir' : e.type === '-' ? 'file' : 'other',
|
|
44
46
|
size: e.size ?? 0,
|
|
45
47
|
mtime: e.modifyTime ? new Date(e.modifyTime) : null
|
|
46
48
|
}))
|
|
47
49
|
}
|
|
48
50
|
|
|
49
51
|
async ensureDir(remotePath) {
|
|
50
|
-
if (!remotePath) return
|
|
52
|
+
if (!remotePath || this.preparedDirs.has(remotePath)) return
|
|
51
53
|
const type = await this.client.exists(remotePath)
|
|
52
54
|
if (type && type !== 'd') throw new Error(`Remote directory path is occupied by a file: ${remotePath}`)
|
|
53
|
-
if (!type)
|
|
55
|
+
if (!type) {
|
|
56
|
+
try { await this.client.mkdir(remotePath, true) }
|
|
57
|
+
catch (error) {
|
|
58
|
+
// Another upload worker may have created the same directory meanwhile.
|
|
59
|
+
if (await this.client.exists(remotePath) !== 'd') throw error
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
this.preparedDirs.add(remotePath)
|
|
54
63
|
}
|
|
55
64
|
|
|
56
65
|
async exists(remotePath) {
|
package/src/ui/deployments.js
CHANGED
|
@@ -24,7 +24,7 @@ export class DeploymentPanel {
|
|
|
24
24
|
this.settingsDryRun = ['true', '1', true].includes(configuration.values.DEVSYNC_DRY_RUN)
|
|
25
25
|
this.configured = !Object.keys(configuration.errors).length
|
|
26
26
|
const element = document.querySelector('#deploymentPanel')
|
|
27
|
-
element.innerHTML = `<h2>Deploy your project</h2><p class="muted">Sync checks local file sizes and timestamps against your saved sync records. Deploy uploads all eligible files. Server files are never deleted.</p><div class="operation-options"><div class="field"><label for="previewMode">Preview mode</label><select id="previewMode"><option value="sync">Changed files (sync)</option><option value="deploy">All eligible files (deploy)</option></select></div><label class="check"><input id="operationDryRun" type="checkbox" ${this.settingsDryRun ? 'checked disabled' : ''}>Preview only for deploy, sync and watch</label></div>${this.settingsDryRun ? '<p class="help">Preview-only mode is enabled in this project’s settings.</p>' : ''}<div class="operation-actions"><button class="secondary" id="previewUploads">Preview uploads</button><button class="secondary" id="deployFiles">Deploy all files</button><button class="primary" id="syncFiles">Sync changes</button><button class="secondary" id="startWatching">Start watching</button><button class="danger" id="stopWatching" hidden>Stop watching</button></div><div class="sync-baseline"><p id="syncBaseline" class="help"></p><button class="secondary" id="scanServer">Scan server / Initial sync</button><p class="help">Use this button only if you are syncing for the first time. It
|
|
27
|
+
element.innerHTML = `<h2>Deploy your project</h2><p class="muted">Sync checks local file sizes and timestamps against your saved sync records. Deploy uploads all eligible files. Server files are never deleted. Up to ${Number(configuration.values.DEVSYNC_CONCURRENCY) || 3} upload queues share the work and retry interrupted files.</p><div class="operation-options"><div class="field"><label for="previewMode">Preview mode</label><select id="previewMode"><option value="sync">Changed files (sync)</option><option value="deploy">All eligible files (deploy)</option></select></div><label class="check"><input id="operationDryRun" type="checkbox" ${this.settingsDryRun ? 'checked disabled' : ''}>Preview only for deploy, sync and watch</label></div>${this.settingsDryRun ? '<p class="help">Preview-only mode is enabled in this project’s settings.</p>' : ''}<div class="operation-actions"><button class="secondary" id="previewUploads">Preview uploads</button><button class="secondary" id="deployFiles">Deploy all files</button><button class="primary" id="syncFiles">Sync changes</button><button class="secondary" id="startWatching">Start watching</button><button class="danger" id="stopWatching" hidden>Stop watching</button></div><div class="sync-baseline"><p id="syncBaseline" class="help"></p><button class="secondary" id="scanServer">Scan server / Initial sync</button><p class="help">Use this button only if you are syncing for the first time. It lists server folders first, then checks local files and saves verified matches without uploading. After that, use Sync changes.</p></div><p class="help">Watch syncs current files first, then follows saves while devsync is running. Stop waits for the current batch to finish.</p><div id="operationStatus" class="notice" role="status"></div><div class="operation-progress"><progress id="uploadProgress" max="1" value="0" aria-label="Operation progress"></progress><span id="progressText" class="help"></span></div><div id="uploadQueues" class="upload-queues" aria-label="Upload queues"></div><div id="operationResult"></div><div id="operationPlan"></div><details id="activityDetails"><summary>Activity log</summary><ol class="operation-log" id="operationLogs"><li>No operations yet.</li></ol></details><p id="lastSync" class="help"></p>`
|
|
28
28
|
document.querySelector('#previewUploads').onclick = () => this.start('preview')
|
|
29
29
|
document.querySelector('#deployFiles').onclick = () => this.start('deploy')
|
|
30
30
|
document.querySelector('#syncFiles').onclick = () => this.start('sync')
|
|
@@ -93,7 +93,7 @@ export class DeploymentPanel {
|
|
|
93
93
|
document.querySelector('#stopWatching').hidden = !startingWatch
|
|
94
94
|
this.disable('stopWatching', !startingWatch || activity.stopping)
|
|
95
95
|
const status = document.querySelector('#operationStatus')
|
|
96
|
-
const phases = { connecting: 'Connecting to hosting…', planning: activity.kind === 'scan' ? '
|
|
96
|
+
const phases = { connecting: 'Connecting to hosting…', inventory: 'Listing server folders…', planning: activity.kind === 'scan' ? 'Comparing local files and saving verified matches…' : 'Checking local files against saved sync records…', uploading: activity.dryRun ? 'Building the preview…' : 'Uploading files…', watching: 'Watching for local saves.', stopping: 'Stopping watching — finishing the current batch…', complete: 'Operation complete.', error: 'Operation needs attention.', idle: 'Preview uploads before deploying, or start a sync when you are ready.' }
|
|
97
97
|
let text = activity.stopping ? phases.stopping : phases[activity.phase] || phases.idle
|
|
98
98
|
if (activity.state === 'success') text = activity.dryRun ? 'Preview complete. No files were uploaded.' : activity.result?.uploaded === 0 ? 'Up to date. No uploads needed.' : 'Upload complete.'
|
|
99
99
|
if (activity.kind === 'scan' && activity.state === 'success') text = `Server scan complete. ${activity.result?.matched || 0} files matched; ${activity.plan?.uploadCount || 0} new or changed. Use Sync changes to upload them.`
|
|
@@ -112,6 +112,8 @@ export class DeploymentPanel {
|
|
|
112
112
|
document.querySelector('#syncBaseline').textContent = baseline?.ready
|
|
113
113
|
? `${baseline.trackedFiles} files tracked for this source and destination. Sync changes uses these records.${baseline.scannedAt ? ' Last server scan: ' + new Date(baseline.scannedAt).toLocaleString() + '.' : ''}`
|
|
114
114
|
: 'No sync records yet for this source and destination. If files are already on your server, scan once to avoid re-uploading matches. Otherwise, Sync changes uploads all eligible files and starts tracking them.'
|
|
115
|
+
const queueNames = { connecting: 'Connecting', uploading: 'Uploading', failed: 'Unavailable · work returned to queue', complete: 'Finished' }
|
|
116
|
+
document.querySelector('#uploadQueues').innerHTML = (activity.workers || []).sort((a, b) => a.workerId - b.workerId).map(worker => `<div class="upload-queue queue-${esc(worker.state)}"><strong>Queue ${worker.workerId}</strong><span>${esc(queueNames[worker.state] || worker.state)}</span>${worker.file ? `<small title="${esc(worker.file)}">${esc(worker.file)}</small>` : ''}</div>`).join('')
|
|
115
117
|
const result = activity.result
|
|
116
118
|
document.querySelector('#operationResult').innerHTML = result?.scanned !== undefined ? `<div class="operation-metrics"><div><strong>${result.scanned}</strong><span>files scanned</span></div><div><strong>${result.matched}</strong><span>matching files</span></div><div><strong>${result.planned}</strong><span>pending upload</span></div></div>` : result ? `<div class="operation-metrics"><div><strong>${result.dryRun ? result.planned : result.uploaded}</strong><span>${result.dryRun ? 'would upload' : 'uploaded'}</span></div><div><strong>${result.failedCount ?? result.failed.length}</strong><span>failed</span></div><div><strong>${bytes(result.bytes)}</strong><span>uploaded</span></div></div>${result.failed.length ? `<details open><summary>Files needing attention</summary><ul class="errors">${result.failed.map((file) => `<li>${esc(file.rel)} — ${esc(file.error)}</li>`).join('')}</ul></details>` : ''}` : ''
|
|
117
119
|
this.renderPlan(activity.plan)
|
package/src/ui/style.css
CHANGED
|
@@ -145,3 +145,5 @@ h1{font-size:25px}
|
|
|
145
145
|
@media(max-width:580px){.brand{font-size:21px;gap:9px}.brand-logo{width:34px;height:34px}.topbar{gap:12px}.connection-pill{font-size:11px;padding:4px 9px}}
|
|
146
146
|
|
|
147
147
|
.about-brand{flex-wrap:wrap}.app-version{font-size:12px;line-height:1.5;font-weight:600;letter-spacing:0;color:var(--accent);background:#203e38;border:1px solid #33554e;border-radius:999px;padding:3px 10px}
|
|
148
|
+
|
|
149
|
+
.upload-queues{display:grid;grid-template-columns:repeat(auto-fit,minmax(160px,1fr));gap:10px;margin:16px 0}.upload-queues:empty{display:none}.upload-queue{min-width:0;padding:12px;border:1px solid var(--line);border-radius:8px;background:#10161f}.upload-queue strong,.upload-queue span,.upload-queue small{display:block}.upload-queue span{font-size:12px;color:var(--muted);margin-top:4px}.upload-queue small{overflow:hidden;text-overflow:ellipsis;white-space:nowrap;color:var(--muted);margin-top:6px}.queue-uploading{border-color:#397367}.queue-uploading strong{color:var(--accent)}.queue-failed span{color:var(--danger)}
|
package/src/ui/ui.js
CHANGED
|
@@ -122,7 +122,7 @@ function renderWizard() {
|
|
|
122
122
|
} else if (step === 2) {
|
|
123
123
|
const transport = draft.values.DEVSYNC_TRANSPORT
|
|
124
124
|
const loaded = Object.entries(draft.sources).filter(([, value]) => value === '.env' || value === '.devsync.env').length
|
|
125
|
-
content = `<h2>Connect your hosting account</h2><p>${loaded ? `Loaded ${loaded} hosting settings from this folder’s environment files. Fill in anything missing.` : 'Enter the connection details from your hosting provider.'} Imported passwords stay on this computer and are not sent to the browser.</p><form id="hostingForm" autocomplete="off"><div class="form-grid"><div class="field"><label for="DEVSYNC_TRANSPORT">Protocol${source('DEVSYNC_TRANSPORT')}</label><select id="DEVSYNC_TRANSPORT" name="DEVSYNC_TRANSPORT"><option value="">Choose a protocol</option>${['ftps', 'sftp', 'ftp'].map((value) => `<option value="${value}" ${transport === value ? 'selected' : ''}>${value === 'ftps' ? 'FTPS — encrypted FTP' : value === 'sftp' ? 'SFTP — SSH file transfer' : 'FTP — unencrypted'}</option>`).join('')}</select></div>${input('DEVSYNC_HOST', 'Host')}${input('DEVSYNC_PORT', 'Port', { type: 'number', optional: true })}${input('DEVSYNC_USERNAME', 'Username')}${input('DEVSYNC_PASSWORD', 'Password / key passphrase', { secret: true })}</div><details><summary>SSH key and advanced settings</summary><div class="form-grid">${input('DEVSYNC_PRIVATE_KEY', 'SSH private key path', { secret: true, optional: true })}<div class="field"><label for="DEVSYNC_DRY_RUN">Default operation</label><select id="DEVSYNC_DRY_RUN" name="DEVSYNC_DRY_RUN"><option value="false" ${String(draft.values.DEVSYNC_DRY_RUN) === 'false' ? 'selected' : ''}>Allow uploads when requested</option><option value="true" ${String(draft.values.DEVSYNC_DRY_RUN) === 'true' ? 'selected' : ''}>Preview only</option></select></div>${input('DEVSYNC_DEBOUNCE_MS', 'Watch delay (milliseconds)', { type: 'number', optional: true })}</div><p class="help">An SSH key is optional and applies to SFTP only. With a key, the password field is an optional passphrase. Local deletions are never mirrored.</p></details><div class="actions"><button class="secondary" type="button" id="backSetup">Back</button><button class="secondary" type="button" id="cancelSetup">Cancel</button><span class="spacer"></span><button class="primary" type="submit">Test connection & continue</button></div></form>`
|
|
125
|
+
content = `<h2>Connect your hosting account</h2><p>${loaded ? `Loaded ${loaded} hosting settings from this folder’s environment files. Fill in anything missing.` : 'Enter the connection details from your hosting provider.'} Imported passwords stay on this computer and are not sent to the browser.</p><form id="hostingForm" autocomplete="off"><div class="form-grid"><div class="field"><label for="DEVSYNC_TRANSPORT">Protocol${source('DEVSYNC_TRANSPORT')}</label><select id="DEVSYNC_TRANSPORT" name="DEVSYNC_TRANSPORT"><option value="">Choose a protocol</option>${['ftps', 'sftp', 'ftp'].map((value) => `<option value="${value}" ${transport === value ? 'selected' : ''}>${value === 'ftps' ? 'FTPS — encrypted FTP' : value === 'sftp' ? 'SFTP — SSH file transfer' : 'FTP — unencrypted'}</option>`).join('')}</select></div>${input('DEVSYNC_HOST', 'Host')}${input('DEVSYNC_PORT', 'Port', { type: 'number', optional: true })}${input('DEVSYNC_USERNAME', 'Username')}${input('DEVSYNC_PASSWORD', 'Password / key passphrase', { secret: true })}</div><details><summary>SSH key and advanced settings</summary><div class="form-grid">${input('DEVSYNC_PRIVATE_KEY', 'SSH private key path', { secret: true, optional: true })}<div class="field"><label for="DEVSYNC_DRY_RUN">Default operation</label><select id="DEVSYNC_DRY_RUN" name="DEVSYNC_DRY_RUN"><option value="false" ${String(draft.values.DEVSYNC_DRY_RUN) === 'false' ? 'selected' : ''}>Allow uploads when requested</option><option value="true" ${String(draft.values.DEVSYNC_DRY_RUN) === 'true' ? 'selected' : ''}>Preview only</option></select></div>${input('DEVSYNC_CONCURRENCY', 'Parallel uploads (1–8)', { type: 'number', optional: true })}${input('DEVSYNC_DEBOUNCE_MS', 'Watch delay (milliseconds)', { type: 'number', optional: true })}</div><p class="help">An SSH key is optional and applies to SFTP only. With a key, the password field is an optional passphrase. Parallel uploads defaults to 3; choose 1 if your host limits connections. Local deletions are never mirrored.</p></details><div class="actions"><button class="secondary" type="button" id="backSetup">Back</button><button class="secondary" type="button" id="cancelSetup">Cancel</button><span class="spacer"></span><button class="primary" type="submit">Test connection & continue</button></div></form>`
|
|
126
126
|
} else if (step === 3) {
|
|
127
127
|
const selected = draft.values.DEVSYNC_REMOTE_PATH
|
|
128
128
|
content = `<h2>Choose the folder on your server</h2><p>Open a folder, then select “Use server folder”. Your local files will be placed inside this destination.</p><div class="notice success">Connection verified with ${esc(draft.values.DEVSYNC_HOST)}.</div><div class="selection"><small>Selected destination ${source('DEVSYNC_REMOTE_PATH')}</small><strong id="selectedRemote">${esc(selected || 'Choose a server folder below')}</strong></div><div id="remotePicker"></div><div class="actions"><button class="secondary" type="button" id="backSetup">Back</button><button class="secondary" type="button" id="cancelSetup">Cancel</button><span class="spacer"></span><button class="primary" id="reviewSetup" ${selected && state.verifiedPath === selected ? '' : 'disabled'}>Continue to review</button></div>`
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
import { setTimeout as delay } from 'node:timers/promises'
|
|
2
|
+
import { isTransient } from './transport/reliable.js'
|
|
3
|
+
|
|
4
|
+
/** Balance by bytes, then let idle workers steal pending work from other lanes. */
|
|
5
|
+
function balance(jobs, count) {
|
|
6
|
+
const lanes = Array.from({ length: count }, () => ({ jobs: [], bytes: 0 }))
|
|
7
|
+
for (const job of [...jobs].sort((a, b) => b.item.bytes - a.item.bytes || a.order - b.order)) {
|
|
8
|
+
const lane = lanes.reduce((best, next) => next.bytes < best.bytes || next.bytes === best.bytes && next.jobs.length < best.jobs.length ? next : best)
|
|
9
|
+
lane.jobs.push(job)
|
|
10
|
+
lane.bytes += job.item.bytes
|
|
11
|
+
}
|
|
12
|
+
return lanes.map(lane => lane.jobs.reverse())
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
/** Each worker owns a connection. A failed connection never strands its files. */
|
|
16
|
+
export async function runUploadQueue(items, {
|
|
17
|
+
createWorker, transport, upload, succeeded = async () => {}, concurrency = 3,
|
|
18
|
+
retries = 2, retryDelayMs = 250, onProgress, onFileResult, onRetry, onWorkerState
|
|
19
|
+
}) {
|
|
20
|
+
if (!Number.isInteger(concurrency) || concurrency < 1 || concurrency > 8) throw new Error('Parallel uploads must be an integer from 1 to 8')
|
|
21
|
+
if (!Number.isInteger(retries) || retries < 0 || retries > 5) throw new Error('Upload retries must be an integer from 0 to 5')
|
|
22
|
+
const total = items.length
|
|
23
|
+
let pending = items.map((item, order) => ({ item, order, attempts: 0 }))
|
|
24
|
+
const done = new Map(), failed = new Map()
|
|
25
|
+
let bytes = 0
|
|
26
|
+
let lastError = new Error('No upload connection is available')
|
|
27
|
+
const finish = (job, error, workerId) => {
|
|
28
|
+
if (error) failed.set(job.order, { rel: job.item.rel, error: error.message, retryable: isTransient(error) })
|
|
29
|
+
else { done.set(job.order, { rel: job.item.rel, bytes: job.item.bytes }); bytes += job.item.bytes }
|
|
30
|
+
onFileResult?.({ item: job.item, index: done.size + failed.size - 1, total, workerId,
|
|
31
|
+
success: !error, ...(error ? { error: error.message } : {}), dryRun: false, bytes })
|
|
32
|
+
}
|
|
33
|
+
// Replacement rounds are bounded too: an outage cannot reconnect forever.
|
|
34
|
+
for (let round = 0; pending.length && round <= retries; round++) {
|
|
35
|
+
if (round) await delay(retryDelayMs * 2 ** (round - 1))
|
|
36
|
+
const count = Math.min(concurrency, pending.length)
|
|
37
|
+
const lanes = balance(pending, count)
|
|
38
|
+
const returned = []
|
|
39
|
+
let retryCursor = 0
|
|
40
|
+
let recoverable = false
|
|
41
|
+
const take = (id) => {
|
|
42
|
+
if (lanes[id].length) return lanes[id].pop()
|
|
43
|
+
const donor = lanes.reduce((best, lane) => lane.length > best.length ? lane : best, [])
|
|
44
|
+
if (donor.length) return donor.pop()
|
|
45
|
+
return retryCursor < returned.length ? returned[retryCursor++] : undefined
|
|
46
|
+
}
|
|
47
|
+
const requeue = (job, error, workerId, connectionFailure = false) => {
|
|
48
|
+
lastError = error
|
|
49
|
+
recoverable ||= isTransient(error)
|
|
50
|
+
job.error = error
|
|
51
|
+
returned.push(job)
|
|
52
|
+
onRetry?.({ item: job.item, error: error.message, workerId,
|
|
53
|
+
attempt: connectionFailure ? round + 1 : job.attempts, retries, connectionFailure })
|
|
54
|
+
}
|
|
55
|
+
const workers = await Promise.allSettled(lanes.map(async (_, id) => {
|
|
56
|
+
let client = round === 0 && id === 0 && transport && !transport.closed ? transport : null
|
|
57
|
+
const borrowed = !!client
|
|
58
|
+
let retired = false
|
|
59
|
+
try {
|
|
60
|
+
for (let job; (job = take(id));) {
|
|
61
|
+
if (!client) {
|
|
62
|
+
onWorkerState?.({ workerId: id + 1, state: 'connecting', file: job.item.rel })
|
|
63
|
+
try { client = await createWorker({ workerId: id + 1 }) }
|
|
64
|
+
catch (error) {
|
|
65
|
+
retired = true
|
|
66
|
+
onWorkerState?.({ workerId: id + 1, state: 'failed', file: job.item.rel })
|
|
67
|
+
requeue(job, error, id + 1, true)
|
|
68
|
+
return
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
onWorkerState?.({ workerId: id + 1, state: 'uploading', file: job.item.rel })
|
|
72
|
+
job.attempts++
|
|
73
|
+
onProgress?.({ phase: 'upload', item: job.item, index: done.size + failed.size, total, workerId: id + 1, workers: count, attempt: job.attempts })
|
|
74
|
+
try {
|
|
75
|
+
await upload(client, job.item)
|
|
76
|
+
} catch (error) {
|
|
77
|
+
if (isTransient(error)) {
|
|
78
|
+
retired = true
|
|
79
|
+
onWorkerState?.({ workerId: id + 1, state: 'failed', file: job.item.rel })
|
|
80
|
+
recoverable = true
|
|
81
|
+
lastError = error
|
|
82
|
+
if (job.attempts <= retries) requeue(job, error, id + 1)
|
|
83
|
+
else finish(job, error, id + 1)
|
|
84
|
+
return
|
|
85
|
+
}
|
|
86
|
+
finish(job, error, id + 1)
|
|
87
|
+
continue
|
|
88
|
+
}
|
|
89
|
+
try { await succeeded(job.item) }
|
|
90
|
+
catch (error) { finish(job, error, id + 1); continue }
|
|
91
|
+
finish(job, null, id + 1)
|
|
92
|
+
}
|
|
93
|
+
} finally {
|
|
94
|
+
if (client && (!borrowed || retired)) await client.close().catch(() => {})
|
|
95
|
+
if (!retired) onWorkerState?.({ workerId: id + 1, state: 'complete', file: null })
|
|
96
|
+
}
|
|
97
|
+
}))
|
|
98
|
+
const crashed = workers.find(worker => worker.status === 'rejected')
|
|
99
|
+
if (crashed) throw crashed.reason
|
|
100
|
+
pending = [...lanes.flat(), ...returned.slice(retryCursor)]
|
|
101
|
+
if (!recoverable) break
|
|
102
|
+
}
|
|
103
|
+
for (const job of pending) finish(job, job.error || lastError)
|
|
104
|
+
const ordered = map => [...map].sort(([a], [b]) => a - b).map(([, value]) => value)
|
|
105
|
+
return { done: ordered(done), failed: ordered(failed), bytes }
|
|
106
|
+
}
|
package/src/uploader.js
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
import fs from 'node:fs'
|
|
2
|
+
import { randomUUID } from 'node:crypto'
|
|
3
|
+
import { runUploadQueue } from './uploadQueue.js'
|
|
2
4
|
import path from 'node:path'
|
|
3
5
|
import { validateRemotePath, relativeFile } from './paths.js'
|
|
4
6
|
import { hashFile } from './checksum.js'
|
|
@@ -6,6 +8,7 @@ import { fingerprint, sameFile } from './core/fileFingerprint.js'
|
|
|
6
8
|
import { isTransient } from './transport/reliable.js'
|
|
7
9
|
|
|
8
10
|
const TMP_SUFFIX = '.devsync-uploading'
|
|
11
|
+
const temporaryPaths = new WeakMap()
|
|
9
12
|
|
|
10
13
|
/** Local Windows path -> remote POSIX path, relative to the project root. */
|
|
11
14
|
export function toRemotePath(projectRoot, remoteRoot, localAbsPath) {
|
|
@@ -59,10 +62,11 @@ export function collectFiles(projectRoot, matcher, { includeIgnored = false, onE
|
|
|
59
62
|
* Build the list of uploads without touching the network for writes.
|
|
60
63
|
* Saved projects compare local metadata with SQLite. Explicit scans verify remote content.
|
|
61
64
|
*/
|
|
62
|
-
export async function buildPlan(project, matcher, transport, { force = false,
|
|
65
|
+
export async function buildPlan(project, matcher, transport, { force = false, remoteInventory, onProgress } = {}) {
|
|
63
66
|
const { localPath, remotePath } = project
|
|
64
|
-
validateRemotePath(remotePath)
|
|
65
|
-
|
|
67
|
+
const root = validateRemotePath(remotePath)
|
|
68
|
+
const realRoot = fs.realpathSync(localPath)
|
|
69
|
+
if (!fs.statSync(realRoot).isDirectory()) throw new Error('Local source must be a directory')
|
|
66
70
|
const excluded = []
|
|
67
71
|
let excludedCount = 0
|
|
68
72
|
const files = collectFiles(localPath, matcher, { onExcluded: (entry) => {
|
|
@@ -70,18 +74,18 @@ export async function buildPlan(project, matcher, transport, { force = false, sc
|
|
|
70
74
|
if (excluded.length < 200) excluded.push(entry)
|
|
71
75
|
} })
|
|
72
76
|
const scope = project.syncScope
|
|
73
|
-
const saved = scope && !
|
|
77
|
+
const saved = scope && !remoteInventory ? (await import('./core/syncState.js')).readSyncFiles(scope) : null
|
|
74
78
|
const matches = []
|
|
75
79
|
const uploads = []
|
|
76
80
|
const skipped = []
|
|
77
81
|
let comparedBytes = 0
|
|
78
82
|
|
|
79
83
|
for (const [index, abs] of files.entries()) {
|
|
80
|
-
const rel =
|
|
81
|
-
const remote =
|
|
84
|
+
const rel = relativeFile(localPath, abs)
|
|
85
|
+
const remote = `${root === '/' ? '' : root}/${rel}`
|
|
82
86
|
let localStat
|
|
83
87
|
try {
|
|
84
|
-
relativeFile(
|
|
88
|
+
relativeFile(realRoot, fs.realpathSync(abs))
|
|
85
89
|
localStat = fs.statSync(abs)
|
|
86
90
|
} catch (err) {
|
|
87
91
|
if (err.code === 'ENOENT') continue
|
|
@@ -91,19 +95,18 @@ export async function buildPlan(project, matcher, transport, { force = false, sc
|
|
|
91
95
|
if (onProgress) onProgress({ phase: 'planning', item: { rel, remote, bytes: localStat.size }, index, total: files.length })
|
|
92
96
|
|
|
93
97
|
const localFingerprint = fingerprint(localStat)
|
|
94
|
-
const item = { abs, rel, remote, bytes: localStat.size, dirs: ancestorsOf(remote), fingerprint: localFingerprint }
|
|
95
98
|
if (saved) {
|
|
96
99
|
if (!force && sameFile(saved.get(rel), localFingerprint)) {
|
|
97
100
|
skipped.push({ rel, bytes: localStat.size, reason: 'unchanged since last successful sync' })
|
|
98
101
|
continue
|
|
99
102
|
}
|
|
100
103
|
} else {
|
|
101
|
-
const remoteStat = await transport.stat(remote)
|
|
104
|
+
const remoteStat = remoteInventory ? remoteInventory.get(rel) : await transport.stat(remote)
|
|
102
105
|
if (remoteStat?.isDir) throw new Error(`Remote file destination is a directory: ${remote}`)
|
|
103
106
|
if (!force && remoteStat && remoteStat.size === localStat.size &&
|
|
104
107
|
await transport.checksum(remote) === await hashFile(abs)) {
|
|
105
108
|
skipped.push({ rel, bytes: localStat.size, reason: 'same content' })
|
|
106
|
-
matches.push(
|
|
109
|
+
matches.push({ abs, rel, fingerprint: localFingerprint })
|
|
107
110
|
continue
|
|
108
111
|
}
|
|
109
112
|
}
|
|
@@ -125,28 +128,47 @@ export async function buildPlan(project, matcher, transport, { force = false, sc
|
|
|
125
128
|
/** Scan server content without uploading; only confirmed matches become the baseline. */
|
|
126
129
|
export async function scanServer(project, matcher, transport, options = {}) {
|
|
127
130
|
if (!project.syncScope) throw new Error('A saved project is required for a server scan')
|
|
128
|
-
const
|
|
131
|
+
const root = validateRemotePath(project.remotePath)
|
|
132
|
+
const remoteInventory = new Map()
|
|
133
|
+
const folders = ['']
|
|
134
|
+
for (let index = 0; index < folders.length; index++) {
|
|
135
|
+
const folder = folders[index]
|
|
136
|
+
const remote = path.posix.join(root, folder)
|
|
137
|
+
options.onProgress?.({ phase: 'inventory', item: { rel: remote }, index, total: folders.length })
|
|
138
|
+
for (const entry of await transport.list(remote)) {
|
|
139
|
+
if (!entry.name || entry.name === '.' || entry.name === '..' || /[\\/\x00-\x1f\x7f]/.test(entry.name)) continue
|
|
140
|
+
const rel = folder ? folder + '/' + entry.name : entry.name
|
|
141
|
+
const isDir = entry.type === 'dir'
|
|
142
|
+
if (entry.type !== 'file' && !isDir) continue
|
|
143
|
+
remoteInventory.set(rel, { isDir, size: entry.size })
|
|
144
|
+
if (isDir && matcher.shouldWatch(path.join(project.localPath, ...rel.split('/')))) folders.push(rel)
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
const plan = await buildPlan(project, matcher, transport, { ...options, force: false, remoteInventory })
|
|
129
148
|
const { replaceScannedFiles } = await import('./core/syncState.js')
|
|
130
149
|
replaceScannedFiles(project.syncScope, plan.matches)
|
|
131
150
|
return plan
|
|
132
151
|
}
|
|
133
152
|
|
|
134
153
|
/**
|
|
135
|
-
* Upload one file atomically: write to
|
|
154
|
+
* Upload one file atomically: write to a unique temporary file, then rename.
|
|
136
155
|
* Without this, PHP can parse a half-written file and 500 mid-transfer.
|
|
137
156
|
*/
|
|
138
|
-
export async function uploadOne(transport, item) {
|
|
139
|
-
if (transport.transaction) return transport.transaction((raw) => uploadAttempt(raw, item))
|
|
157
|
+
export async function uploadOne(transport, item, options) {
|
|
158
|
+
if (transport.transaction) return transport.transaction((raw) => uploadAttempt(raw, item), options)
|
|
140
159
|
return uploadAttempt(transport, item)
|
|
141
160
|
}
|
|
142
161
|
|
|
143
162
|
async function uploadAttempt(transport, item) {
|
|
144
|
-
|
|
145
|
-
|
|
163
|
+
const parent = item.dirs.at(-1)
|
|
164
|
+
if (parent) await transport.ensureDir(parent)
|
|
165
|
+
// Reuse this file’s temporary name on retry so a leftover partial file is replaced.
|
|
166
|
+
let tmp = temporaryPaths.get(item)
|
|
167
|
+
if (!tmp) {
|
|
168
|
+
tmp = path.posix.join(path.posix.dirname(item.remote), randomUUID() + TMP_SUFFIX)
|
|
169
|
+
temporaryPaths.set(item, tmp)
|
|
146
170
|
}
|
|
147
|
-
const tmp = item.remote + TMP_SUFFIX
|
|
148
171
|
try {
|
|
149
|
-
await transport.delete(tmp).catch(() => {})
|
|
150
172
|
await transport.put(item.abs, tmp)
|
|
151
173
|
await transport.rename(tmp, item.remote)
|
|
152
174
|
} catch (err) {
|
|
@@ -155,7 +177,16 @@ async function uploadAttempt(transport, item) {
|
|
|
155
177
|
}
|
|
156
178
|
}
|
|
157
179
|
|
|
158
|
-
export async function runPlan(plan, transport, { dryRun = false, onProgress, onFileResult } = {}) {
|
|
180
|
+
export async function runPlan(plan, transport, { dryRun = false, onProgress, onFileResult, createWorker, concurrency = 3, retries = 2, retryDelayMs = 250, onRetry, onWorkerState } = {}) {
|
|
181
|
+
const record = !dryRun && plan.syncScope && plan.uploads.length
|
|
182
|
+
? (await import('./core/syncState.js')).recordUploadedFile : null
|
|
183
|
+
if (!dryRun && createWorker) {
|
|
184
|
+
return runUploadQueue(plan.uploads, {
|
|
185
|
+
transport, createWorker, concurrency, retries, retryDelayMs, onProgress, onFileResult, onRetry, onWorkerState,
|
|
186
|
+
upload: (client, item) => uploadOne(client, item, { retries: 0 }),
|
|
187
|
+
succeeded: item => record?.(plan.syncScope, item)
|
|
188
|
+
})
|
|
189
|
+
}
|
|
159
190
|
const done = []
|
|
160
191
|
const failed = []
|
|
161
192
|
let bytes = 0
|
|
@@ -171,10 +202,7 @@ export async function runPlan(plan, transport, { dryRun = false, onProgress, onF
|
|
|
171
202
|
}
|
|
172
203
|
try {
|
|
173
204
|
await uploadOne(transport, item)
|
|
174
|
-
|
|
175
|
-
const { recordUploadedFile } = await import('./core/syncState.js')
|
|
176
|
-
recordUploadedFile(plan.syncScope, item)
|
|
177
|
-
}
|
|
205
|
+
record?.(plan.syncScope, item)
|
|
178
206
|
done.push({ rel: item.rel, bytes: item.bytes })
|
|
179
207
|
bytes += item.bytes
|
|
180
208
|
if (onFileResult) onFileResult({ item, index: i, total: plan.uploads.length, success: true, dryRun: false, bytes })
|
package/src/watcher.js
CHANGED
|
@@ -10,6 +10,9 @@ export class ProjectWatcher {
|
|
|
10
10
|
this.matcher = matcher
|
|
11
11
|
this.transport = transport
|
|
12
12
|
this.debounceMs = opts.debounceMs ?? 400
|
|
13
|
+
this.createWorker = opts.createWorker
|
|
14
|
+
this.concurrency = opts.concurrency ?? 3
|
|
15
|
+
this.onWorkerState = opts.onWorkerState
|
|
13
16
|
this.dryRun = opts.dryRun ?? false
|
|
14
17
|
this.onEvent = opts.onEvent ?? (() => {})
|
|
15
18
|
this.onSettle = opts.onSettle ?? (() => {})
|
|
@@ -84,7 +87,9 @@ export class ProjectWatcher {
|
|
|
84
87
|
const plan = await buildPlan(this.project, this.matcher, this.transport, { onProgress: this.onProgress })
|
|
85
88
|
this.onPlan(plan)
|
|
86
89
|
const result = await runPlan(plan, this.transport, {
|
|
87
|
-
dryRun: this.dryRun,
|
|
90
|
+
dryRun: this.dryRun, createWorker: this.createWorker, concurrency: this.concurrency,
|
|
91
|
+
onProgress: this.onProgress, onFileResult: this.onFileResult, onWorkerState: this.onWorkerState,
|
|
92
|
+
onRetry: event => this.onEvent({ type: 'retry', message: `Queue ${event.workerId}: ${event.connectionFailure ? 'connection unavailable' : event.item.rel + ' interrupted'}; returned to the pending queue.` })
|
|
88
93
|
})
|
|
89
94
|
this.stats.cycles++
|
|
90
95
|
if (!this.dryRun) this.stats.uploaded += result.done.length
|
|
@@ -119,4 +124,4 @@ export class ProjectWatcher {
|
|
|
119
124
|
if (this.watcher) await this.watcher.close()
|
|
120
125
|
await this.cycle
|
|
121
126
|
}
|
|
122
|
-
}
|
|
127
|
+
}
|