shared-hosting-sync 0.1.3 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +3 -0
- package/CHANGELOG.md +25 -0
- package/README.md +61 -13
- package/RELEASING.md +7 -7
- package/package.json +2 -2
- package/src/arguments.js +2 -2
- package/src/cli.js +535 -511
- package/src/core/deployment.js +3 -0
- package/src/core/envSchema.js +9 -0
- package/src/core/fileFingerprint.js +9 -0
- package/src/core/migrations/003_sync_state.sql +18 -0
- package/src/core/syncState.js +65 -0
- package/src/server/deployments.js +46 -9
- package/src/server/index.js +2 -1
- package/src/server/routes.js +1 -1
- package/src/transport/ftp.js +1 -1
- package/src/transport/reliable.js +3 -3
- package/src/transport/sftp.js +8 -2
- package/src/ui/deployments.js +17 -7
- package/src/ui/index.html +7 -6
- package/src/ui/logo.svg +7 -0
- package/src/ui/style.css +10 -2
- package/src/ui/ui.js +303 -293
- package/src/uploadQueue.js +106 -0
- package/src/uploader.js +69 -12
- package/src/watcher.js +7 -2
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
import { setTimeout as delay } from 'node:timers/promises'
|
|
2
|
+
import { isTransient } from './transport/reliable.js'
|
|
3
|
+
|
|
4
|
+
/** Balance by bytes, then let idle workers steal pending work from other lanes. */
|
|
5
|
+
function balance(jobs, count) {
|
|
6
|
+
const lanes = Array.from({ length: count }, () => ({ jobs: [], bytes: 0 }))
|
|
7
|
+
for (const job of [...jobs].sort((a, b) => b.item.bytes - a.item.bytes || a.order - b.order)) {
|
|
8
|
+
const lane = lanes.reduce((best, next) => next.bytes < best.bytes || next.bytes === best.bytes && next.jobs.length < best.jobs.length ? next : best)
|
|
9
|
+
lane.jobs.push(job)
|
|
10
|
+
lane.bytes += job.item.bytes
|
|
11
|
+
}
|
|
12
|
+
return lanes.map(lane => lane.jobs.reverse())
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
/** Each worker owns a connection. A failed connection never strands its files. */
|
|
16
|
+
export async function runUploadQueue(items, {
|
|
17
|
+
createWorker, transport, upload, succeeded = async () => {}, concurrency = 3,
|
|
18
|
+
retries = 2, retryDelayMs = 250, onProgress, onFileResult, onRetry, onWorkerState
|
|
19
|
+
}) {
|
|
20
|
+
if (!Number.isInteger(concurrency) || concurrency < 1 || concurrency > 8) throw new Error('Parallel uploads must be an integer from 1 to 8')
|
|
21
|
+
if (!Number.isInteger(retries) || retries < 0 || retries > 5) throw new Error('Upload retries must be an integer from 0 to 5')
|
|
22
|
+
const total = items.length
|
|
23
|
+
let pending = items.map((item, order) => ({ item, order, attempts: 0 }))
|
|
24
|
+
const done = new Map(), failed = new Map()
|
|
25
|
+
let bytes = 0
|
|
26
|
+
let lastError = new Error('No upload connection is available')
|
|
27
|
+
const finish = (job, error, workerId) => {
|
|
28
|
+
if (error) failed.set(job.order, { rel: job.item.rel, error: error.message, retryable: isTransient(error) })
|
|
29
|
+
else { done.set(job.order, { rel: job.item.rel, bytes: job.item.bytes }); bytes += job.item.bytes }
|
|
30
|
+
onFileResult?.({ item: job.item, index: done.size + failed.size - 1, total, workerId,
|
|
31
|
+
success: !error, ...(error ? { error: error.message } : {}), dryRun: false, bytes })
|
|
32
|
+
}
|
|
33
|
+
// Replacement rounds are bounded too: an outage cannot reconnect forever.
|
|
34
|
+
for (let round = 0; pending.length && round <= retries; round++) {
|
|
35
|
+
if (round) await delay(retryDelayMs * 2 ** (round - 1))
|
|
36
|
+
const count = Math.min(concurrency, pending.length)
|
|
37
|
+
const lanes = balance(pending, count)
|
|
38
|
+
const returned = []
|
|
39
|
+
let retryCursor = 0
|
|
40
|
+
let recoverable = false
|
|
41
|
+
const take = (id) => {
|
|
42
|
+
if (lanes[id].length) return lanes[id].pop()
|
|
43
|
+
const donor = lanes.reduce((best, lane) => lane.length > best.length ? lane : best, [])
|
|
44
|
+
if (donor.length) return donor.pop()
|
|
45
|
+
return retryCursor < returned.length ? returned[retryCursor++] : undefined
|
|
46
|
+
}
|
|
47
|
+
const requeue = (job, error, workerId, connectionFailure = false) => {
|
|
48
|
+
lastError = error
|
|
49
|
+
recoverable ||= isTransient(error)
|
|
50
|
+
job.error = error
|
|
51
|
+
returned.push(job)
|
|
52
|
+
onRetry?.({ item: job.item, error: error.message, workerId,
|
|
53
|
+
attempt: connectionFailure ? round + 1 : job.attempts, retries, connectionFailure })
|
|
54
|
+
}
|
|
55
|
+
const workers = await Promise.allSettled(lanes.map(async (_, id) => {
|
|
56
|
+
let client = round === 0 && id === 0 && transport && !transport.closed ? transport : null
|
|
57
|
+
const borrowed = !!client
|
|
58
|
+
let retired = false
|
|
59
|
+
try {
|
|
60
|
+
for (let job; (job = take(id));) {
|
|
61
|
+
if (!client) {
|
|
62
|
+
onWorkerState?.({ workerId: id + 1, state: 'connecting', file: job.item.rel })
|
|
63
|
+
try { client = await createWorker({ workerId: id + 1 }) }
|
|
64
|
+
catch (error) {
|
|
65
|
+
retired = true
|
|
66
|
+
onWorkerState?.({ workerId: id + 1, state: 'failed', file: job.item.rel })
|
|
67
|
+
requeue(job, error, id + 1, true)
|
|
68
|
+
return
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
onWorkerState?.({ workerId: id + 1, state: 'uploading', file: job.item.rel })
|
|
72
|
+
job.attempts++
|
|
73
|
+
onProgress?.({ phase: 'upload', item: job.item, index: done.size + failed.size, total, workerId: id + 1, workers: count, attempt: job.attempts })
|
|
74
|
+
try {
|
|
75
|
+
await upload(client, job.item)
|
|
76
|
+
} catch (error) {
|
|
77
|
+
if (isTransient(error)) {
|
|
78
|
+
retired = true
|
|
79
|
+
onWorkerState?.({ workerId: id + 1, state: 'failed', file: job.item.rel })
|
|
80
|
+
recoverable = true
|
|
81
|
+
lastError = error
|
|
82
|
+
if (job.attempts <= retries) requeue(job, error, id + 1)
|
|
83
|
+
else finish(job, error, id + 1)
|
|
84
|
+
return
|
|
85
|
+
}
|
|
86
|
+
finish(job, error, id + 1)
|
|
87
|
+
continue
|
|
88
|
+
}
|
|
89
|
+
try { await succeeded(job.item) }
|
|
90
|
+
catch (error) { finish(job, error, id + 1); continue }
|
|
91
|
+
finish(job, null, id + 1)
|
|
92
|
+
}
|
|
93
|
+
} finally {
|
|
94
|
+
if (client && (!borrowed || retired)) await client.close().catch(() => {})
|
|
95
|
+
if (!retired) onWorkerState?.({ workerId: id + 1, state: 'complete', file: null })
|
|
96
|
+
}
|
|
97
|
+
}))
|
|
98
|
+
const crashed = workers.find(worker => worker.status === 'rejected')
|
|
99
|
+
if (crashed) throw crashed.reason
|
|
100
|
+
pending = [...lanes.flat(), ...returned.slice(retryCursor)]
|
|
101
|
+
if (!recoverable) break
|
|
102
|
+
}
|
|
103
|
+
for (const job of pending) finish(job, job.error || lastError)
|
|
104
|
+
const ordered = map => [...map].sort(([a], [b]) => a - b).map(([, value]) => value)
|
|
105
|
+
return { done: ordered(done), failed: ordered(failed), bytes }
|
|
106
|
+
}
|
package/src/uploader.js
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
import fs from 'node:fs'
|
|
2
|
+
import { runUploadQueue } from './uploadQueue.js'
|
|
2
3
|
import path from 'node:path'
|
|
3
4
|
import { validateRemotePath, relativeFile } from './paths.js'
|
|
4
5
|
import { hashFile } from './checksum.js'
|
|
6
|
+
import { fingerprint, sameFile } from './core/fileFingerprint.js'
|
|
5
7
|
import { isTransient } from './transport/reliable.js'
|
|
6
8
|
|
|
7
9
|
const TMP_SUFFIX = '.devsync-uploading'
|
|
@@ -56,9 +58,9 @@ export function collectFiles(projectRoot, matcher, { includeIgnored = false, onE
|
|
|
56
58
|
|
|
57
59
|
/**
|
|
58
60
|
* Build the list of uploads without touching the network for writes.
|
|
59
|
-
*
|
|
61
|
+
* Saved projects compare local metadata with SQLite. Explicit scans verify remote content.
|
|
60
62
|
*/
|
|
61
|
-
export async function buildPlan(project, matcher, transport, { force = false, onProgress } = {}) {
|
|
63
|
+
export async function buildPlan(project, matcher, transport, { force = false, remoteInventory, onProgress } = {}) {
|
|
62
64
|
const { localPath, remotePath } = project
|
|
63
65
|
validateRemotePath(remotePath)
|
|
64
66
|
if (!fs.statSync(localPath).isDirectory()) throw new Error('Local source must be a directory')
|
|
@@ -68,6 +70,9 @@ export async function buildPlan(project, matcher, transport, { force = false, on
|
|
|
68
70
|
excludedCount++
|
|
69
71
|
if (excluded.length < 200) excluded.push(entry)
|
|
70
72
|
} })
|
|
73
|
+
const scope = project.syncScope
|
|
74
|
+
const saved = scope && !remoteInventory ? (await import('./core/syncState.js')).readSyncFiles(scope) : null
|
|
75
|
+
const matches = []
|
|
71
76
|
const uploads = []
|
|
72
77
|
const skipped = []
|
|
73
78
|
let comparedBytes = 0
|
|
@@ -86,12 +91,22 @@ export async function buildPlan(project, matcher, transport, { force = false, on
|
|
|
86
91
|
|
|
87
92
|
if (onProgress) onProgress({ phase: 'planning', item: { rel, remote, bytes: localStat.size }, index, total: files.length })
|
|
88
93
|
|
|
89
|
-
const
|
|
90
|
-
|
|
91
|
-
if (
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
94
|
+
const localFingerprint = fingerprint(localStat)
|
|
95
|
+
const item = { abs, rel, remote, bytes: localStat.size, dirs: ancestorsOf(remote), fingerprint: localFingerprint }
|
|
96
|
+
if (saved) {
|
|
97
|
+
if (!force && sameFile(saved.get(rel), localFingerprint)) {
|
|
98
|
+
skipped.push({ rel, bytes: localStat.size, reason: 'unchanged since last successful sync' })
|
|
99
|
+
continue
|
|
100
|
+
}
|
|
101
|
+
} else {
|
|
102
|
+
const remoteStat = remoteInventory ? remoteInventory.get(rel) : await transport.stat(remote)
|
|
103
|
+
if (remoteStat?.isDir) throw new Error(`Remote file destination is a directory: ${remote}`)
|
|
104
|
+
if (!force && remoteStat && remoteStat.size === localStat.size &&
|
|
105
|
+
await transport.checksum(remote) === await hashFile(abs)) {
|
|
106
|
+
skipped.push({ rel, bytes: localStat.size, reason: 'same content' })
|
|
107
|
+
matches.push(item)
|
|
108
|
+
continue
|
|
109
|
+
}
|
|
95
110
|
}
|
|
96
111
|
|
|
97
112
|
uploads.push({
|
|
@@ -99,20 +114,46 @@ export async function buildPlan(project, matcher, transport, { force = false, on
|
|
|
99
114
|
rel,
|
|
100
115
|
remote,
|
|
101
116
|
bytes: localStat.size,
|
|
117
|
+
fingerprint: localFingerprint,
|
|
102
118
|
dirs: ancestorsOf(remote)
|
|
103
119
|
})
|
|
104
120
|
comparedBytes += localStat.size
|
|
105
121
|
}
|
|
106
122
|
|
|
107
|
-
return { uploads, skipped, excluded, excludedCount, totalFiles: files.length, comparedBytes }
|
|
123
|
+
return { uploads, skipped, excluded, excludedCount, totalFiles: files.length, comparedBytes, syncScope: scope, matches }
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
/** Scan server content without uploading; only confirmed matches become the baseline. */
|
|
127
|
+
export async function scanServer(project, matcher, transport, options = {}) {
|
|
128
|
+
if (!project.syncScope) throw new Error('A saved project is required for a server scan')
|
|
129
|
+
const root = validateRemotePath(project.remotePath)
|
|
130
|
+
const remoteInventory = new Map()
|
|
131
|
+
const folders = ['']
|
|
132
|
+
for (let index = 0; index < folders.length; index++) {
|
|
133
|
+
const folder = folders[index]
|
|
134
|
+
const remote = path.posix.join(root, folder)
|
|
135
|
+
options.onProgress?.({ phase: 'inventory', item: { rel: remote }, index, total: folders.length })
|
|
136
|
+
for (const entry of await transport.list(remote)) {
|
|
137
|
+
if (!entry.name || entry.name === '.' || entry.name === '..' || /[\\/\x00-\x1f\x7f]/.test(entry.name)) continue
|
|
138
|
+
const rel = folder ? folder + '/' + entry.name : entry.name
|
|
139
|
+
const isDir = entry.type === 'dir'
|
|
140
|
+
if (entry.type !== 'file' && !isDir) continue
|
|
141
|
+
remoteInventory.set(rel, { isDir, size: entry.size })
|
|
142
|
+
if (isDir && matcher.shouldWatch(path.join(project.localPath, ...rel.split('/')))) folders.push(rel)
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
const plan = await buildPlan(project, matcher, transport, { ...options, force: false, remoteInventory })
|
|
146
|
+
const { replaceScannedFiles } = await import('./core/syncState.js')
|
|
147
|
+
replaceScannedFiles(project.syncScope, plan.matches)
|
|
148
|
+
return plan
|
|
108
149
|
}
|
|
109
150
|
|
|
110
151
|
/**
|
|
111
152
|
* Upload one file atomically: write to <name>.devsync-uploading, then rename.
|
|
112
153
|
* Without this, PHP can parse a half-written file and 500 mid-transfer.
|
|
113
154
|
*/
|
|
114
|
-
export async function uploadOne(transport, item) {
|
|
115
|
-
if (transport.transaction) return transport.transaction((raw) => uploadAttempt(raw, item))
|
|
155
|
+
export async function uploadOne(transport, item, options) {
|
|
156
|
+
if (transport.transaction) return transport.transaction((raw) => uploadAttempt(raw, item), options)
|
|
116
157
|
return uploadAttempt(transport, item)
|
|
117
158
|
}
|
|
118
159
|
|
|
@@ -131,7 +172,19 @@ async function uploadAttempt(transport, item) {
|
|
|
131
172
|
}
|
|
132
173
|
}
|
|
133
174
|
|
|
134
|
-
export async function runPlan(plan, transport, { dryRun = false, onProgress, onFileResult } = {}) {
|
|
175
|
+
export async function runPlan(plan, transport, { dryRun = false, onProgress, onFileResult, createWorker, concurrency = 3, retries = 2, retryDelayMs = 250, onRetry, onWorkerState } = {}) {
|
|
176
|
+
if (!dryRun && createWorker) {
|
|
177
|
+
return runUploadQueue(plan.uploads, {
|
|
178
|
+
transport, createWorker, concurrency, retries, retryDelayMs, onProgress, onFileResult, onRetry, onWorkerState,
|
|
179
|
+
upload: (client, item) => uploadOne(client, item, { retries: 0 }),
|
|
180
|
+
succeeded: async item => {
|
|
181
|
+
if (plan.syncScope) {
|
|
182
|
+
const { recordUploadedFile } = await import('./core/syncState.js')
|
|
183
|
+
recordUploadedFile(plan.syncScope, item)
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
})
|
|
187
|
+
}
|
|
135
188
|
const done = []
|
|
136
189
|
const failed = []
|
|
137
190
|
let bytes = 0
|
|
@@ -147,6 +200,10 @@ export async function runPlan(plan, transport, { dryRun = false, onProgress, onF
|
|
|
147
200
|
}
|
|
148
201
|
try {
|
|
149
202
|
await uploadOne(transport, item)
|
|
203
|
+
if (plan.syncScope) {
|
|
204
|
+
const { recordUploadedFile } = await import('./core/syncState.js')
|
|
205
|
+
recordUploadedFile(plan.syncScope, item)
|
|
206
|
+
}
|
|
150
207
|
done.push({ rel: item.rel, bytes: item.bytes })
|
|
151
208
|
bytes += item.bytes
|
|
152
209
|
if (onFileResult) onFileResult({ item, index: i, total: plan.uploads.length, success: true, dryRun: false, bytes })
|
package/src/watcher.js
CHANGED
|
@@ -10,6 +10,9 @@ export class ProjectWatcher {
|
|
|
10
10
|
this.matcher = matcher
|
|
11
11
|
this.transport = transport
|
|
12
12
|
this.debounceMs = opts.debounceMs ?? 400
|
|
13
|
+
this.createWorker = opts.createWorker
|
|
14
|
+
this.concurrency = opts.concurrency ?? 3
|
|
15
|
+
this.onWorkerState = opts.onWorkerState
|
|
13
16
|
this.dryRun = opts.dryRun ?? false
|
|
14
17
|
this.onEvent = opts.onEvent ?? (() => {})
|
|
15
18
|
this.onSettle = opts.onSettle ?? (() => {})
|
|
@@ -84,7 +87,9 @@ export class ProjectWatcher {
|
|
|
84
87
|
const plan = await buildPlan(this.project, this.matcher, this.transport, { onProgress: this.onProgress })
|
|
85
88
|
this.onPlan(plan)
|
|
86
89
|
const result = await runPlan(plan, this.transport, {
|
|
87
|
-
dryRun: this.dryRun,
|
|
90
|
+
dryRun: this.dryRun, createWorker: this.createWorker, concurrency: this.concurrency,
|
|
91
|
+
onProgress: this.onProgress, onFileResult: this.onFileResult, onWorkerState: this.onWorkerState,
|
|
92
|
+
onRetry: event => this.onEvent({ type: 'retry', message: `Queue ${event.workerId}: ${event.connectionFailure ? 'connection unavailable' : event.item.rel + ' interrupted'}; returned to the pending queue.` })
|
|
88
93
|
})
|
|
89
94
|
this.stats.cycles++
|
|
90
95
|
if (!this.dryRun) this.stats.uploaded += result.done.length
|
|
@@ -119,4 +124,4 @@ export class ProjectWatcher {
|
|
|
119
124
|
if (this.watcher) await this.watcher.close()
|
|
120
125
|
await this.cycle
|
|
121
126
|
}
|
|
122
|
-
}
|
|
127
|
+
}
|