shared-hosting-sync 0.1.3 → 0.1.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,106 @@
1
+ import { setTimeout as delay } from 'node:timers/promises'
2
+ import { isTransient } from './transport/reliable.js'
3
+
4
+ /** Balance by bytes, then let idle workers steal pending work from other lanes. */
5
+ function balance(jobs, count) {
6
+ const lanes = Array.from({ length: count }, () => ({ jobs: [], bytes: 0 }))
7
+ for (const job of [...jobs].sort((a, b) => b.item.bytes - a.item.bytes || a.order - b.order)) {
8
+ const lane = lanes.reduce((best, next) => next.bytes < best.bytes || next.bytes === best.bytes && next.jobs.length < best.jobs.length ? next : best)
9
+ lane.jobs.push(job)
10
+ lane.bytes += job.item.bytes
11
+ }
12
+ return lanes.map(lane => lane.jobs.reverse())
13
+ }
14
+
15
+ /** Each worker owns a connection. A failed connection never strands its files. */
16
+ export async function runUploadQueue(items, {
17
+ createWorker, transport, upload, succeeded = async () => {}, concurrency = 3,
18
+ retries = 2, retryDelayMs = 250, onProgress, onFileResult, onRetry, onWorkerState
19
+ }) {
20
+ if (!Number.isInteger(concurrency) || concurrency < 1 || concurrency > 8) throw new Error('Parallel uploads must be an integer from 1 to 8')
21
+ if (!Number.isInteger(retries) || retries < 0 || retries > 5) throw new Error('Upload retries must be an integer from 0 to 5')
22
+ const total = items.length
23
+ let pending = items.map((item, order) => ({ item, order, attempts: 0 }))
24
+ const done = new Map(), failed = new Map()
25
+ let bytes = 0
26
+ let lastError = new Error('No upload connection is available')
27
+ const finish = (job, error, workerId) => {
28
+ if (error) failed.set(job.order, { rel: job.item.rel, error: error.message, retryable: isTransient(error) })
29
+ else { done.set(job.order, { rel: job.item.rel, bytes: job.item.bytes }); bytes += job.item.bytes }
30
+ onFileResult?.({ item: job.item, index: done.size + failed.size - 1, total, workerId,
31
+ success: !error, ...(error ? { error: error.message } : {}), dryRun: false, bytes })
32
+ }
33
+ // Replacement rounds are bounded too: an outage cannot reconnect forever.
34
+ for (let round = 0; pending.length && round <= retries; round++) {
35
+ if (round) await delay(retryDelayMs * 2 ** (round - 1))
36
+ const count = Math.min(concurrency, pending.length)
37
+ const lanes = balance(pending, count)
38
+ const returned = []
39
+ let retryCursor = 0
40
+ let recoverable = false
41
+ const take = (id) => {
42
+ if (lanes[id].length) return lanes[id].pop()
43
+ const donor = lanes.reduce((best, lane) => lane.length > best.length ? lane : best, [])
44
+ if (donor.length) return donor.pop()
45
+ return retryCursor < returned.length ? returned[retryCursor++] : undefined
46
+ }
47
+ const requeue = (job, error, workerId, connectionFailure = false) => {
48
+ lastError = error
49
+ recoverable ||= isTransient(error)
50
+ job.error = error
51
+ returned.push(job)
52
+ onRetry?.({ item: job.item, error: error.message, workerId,
53
+ attempt: connectionFailure ? round + 1 : job.attempts, retries, connectionFailure })
54
+ }
55
+ const workers = await Promise.allSettled(lanes.map(async (_, id) => {
56
+ let client = round === 0 && id === 0 && transport && !transport.closed ? transport : null
57
+ const borrowed = !!client
58
+ let retired = false
59
+ try {
60
+ for (let job; (job = take(id));) {
61
+ if (!client) {
62
+ onWorkerState?.({ workerId: id + 1, state: 'connecting', file: job.item.rel })
63
+ try { client = await createWorker({ workerId: id + 1 }) }
64
+ catch (error) {
65
+ retired = true
66
+ onWorkerState?.({ workerId: id + 1, state: 'failed', file: job.item.rel })
67
+ requeue(job, error, id + 1, true)
68
+ return
69
+ }
70
+ }
71
+ onWorkerState?.({ workerId: id + 1, state: 'uploading', file: job.item.rel })
72
+ job.attempts++
73
+ onProgress?.({ phase: 'upload', item: job.item, index: done.size + failed.size, total, workerId: id + 1, workers: count, attempt: job.attempts })
74
+ try {
75
+ await upload(client, job.item)
76
+ } catch (error) {
77
+ if (isTransient(error)) {
78
+ retired = true
79
+ onWorkerState?.({ workerId: id + 1, state: 'failed', file: job.item.rel })
80
+ recoverable = true
81
+ lastError = error
82
+ if (job.attempts <= retries) requeue(job, error, id + 1)
83
+ else finish(job, error, id + 1)
84
+ return
85
+ }
86
+ finish(job, error, id + 1)
87
+ continue
88
+ }
89
+ try { await succeeded(job.item) }
90
+ catch (error) { finish(job, error, id + 1); continue }
91
+ finish(job, null, id + 1)
92
+ }
93
+ } finally {
94
+ if (client && (!borrowed || retired)) await client.close().catch(() => {})
95
+ if (!retired) onWorkerState?.({ workerId: id + 1, state: 'complete', file: null })
96
+ }
97
+ }))
98
+ const crashed = workers.find(worker => worker.status === 'rejected')
99
+ if (crashed) throw crashed.reason
100
+ pending = [...lanes.flat(), ...returned.slice(retryCursor)]
101
+ if (!recoverable) break
102
+ }
103
+ for (const job of pending) finish(job, job.error || lastError)
104
+ const ordered = map => [...map].sort(([a], [b]) => a - b).map(([, value]) => value)
105
+ return { done: ordered(done), failed: ordered(failed), bytes }
106
+ }
package/src/uploader.js CHANGED
@@ -1,7 +1,9 @@
1
1
  import fs from 'node:fs'
2
+ import { runUploadQueue } from './uploadQueue.js'
2
3
  import path from 'node:path'
3
4
  import { validateRemotePath, relativeFile } from './paths.js'
4
5
  import { hashFile } from './checksum.js'
6
+ import { fingerprint, sameFile } from './core/fileFingerprint.js'
5
7
  import { isTransient } from './transport/reliable.js'
6
8
 
7
9
  const TMP_SUFFIX = '.devsync-uploading'
@@ -56,9 +58,9 @@ export function collectFiles(projectRoot, matcher, { includeIgnored = false, onE
56
58
 
57
59
  /**
58
60
  * Build the list of uploads without touching the network for writes.
59
- * Size differences upload immediately; equal sizes require a content match.
61
+ * Saved projects compare local metadata with SQLite. Explicit scans verify remote content.
60
62
  */
61
- export async function buildPlan(project, matcher, transport, { force = false, onProgress } = {}) {
63
+ export async function buildPlan(project, matcher, transport, { force = false, remoteInventory, onProgress } = {}) {
62
64
  const { localPath, remotePath } = project
63
65
  validateRemotePath(remotePath)
64
66
  if (!fs.statSync(localPath).isDirectory()) throw new Error('Local source must be a directory')
@@ -68,6 +70,9 @@ export async function buildPlan(project, matcher, transport, { force = false, on
68
70
  excludedCount++
69
71
  if (excluded.length < 200) excluded.push(entry)
70
72
  } })
73
+ const scope = project.syncScope
74
+ const saved = scope && !remoteInventory ? (await import('./core/syncState.js')).readSyncFiles(scope) : null
75
+ const matches = []
71
76
  const uploads = []
72
77
  const skipped = []
73
78
  let comparedBytes = 0
@@ -86,12 +91,22 @@ export async function buildPlan(project, matcher, transport, { force = false, on
86
91
 
87
92
  if (onProgress) onProgress({ phase: 'planning', item: { rel, remote, bytes: localStat.size }, index, total: files.length })
88
93
 
89
- const remoteStat = await transport.stat(remote)
90
- if (remoteStat?.isDir) throw new Error(`Remote file destination is a directory: ${remote}`)
91
- if (!force && remoteStat && remoteStat.size === localStat.size &&
92
- await transport.checksum(remote) === await hashFile(abs)) {
93
- skipped.push({ rel, bytes: localStat.size, reason: 'same content' })
94
- continue
94
+ const localFingerprint = fingerprint(localStat)
95
+ const item = { abs, rel, remote, bytes: localStat.size, dirs: ancestorsOf(remote), fingerprint: localFingerprint }
96
+ if (saved) {
97
+ if (!force && sameFile(saved.get(rel), localFingerprint)) {
98
+ skipped.push({ rel, bytes: localStat.size, reason: 'unchanged since last successful sync' })
99
+ continue
100
+ }
101
+ } else {
102
+ const remoteStat = remoteInventory ? remoteInventory.get(rel) : await transport.stat(remote)
103
+ if (remoteStat?.isDir) throw new Error(`Remote file destination is a directory: ${remote}`)
104
+ if (!force && remoteStat && remoteStat.size === localStat.size &&
105
+ await transport.checksum(remote) === await hashFile(abs)) {
106
+ skipped.push({ rel, bytes: localStat.size, reason: 'same content' })
107
+ matches.push(item)
108
+ continue
109
+ }
95
110
  }
96
111
 
97
112
  uploads.push({
@@ -99,20 +114,46 @@ export async function buildPlan(project, matcher, transport, { force = false, on
99
114
  rel,
100
115
  remote,
101
116
  bytes: localStat.size,
117
+ fingerprint: localFingerprint,
102
118
  dirs: ancestorsOf(remote)
103
119
  })
104
120
  comparedBytes += localStat.size
105
121
  }
106
122
 
107
- return { uploads, skipped, excluded, excludedCount, totalFiles: files.length, comparedBytes }
123
+ return { uploads, skipped, excluded, excludedCount, totalFiles: files.length, comparedBytes, syncScope: scope, matches }
124
+ }
125
+
126
+ /** Scan server content without uploading; only confirmed matches become the baseline. */
127
+ export async function scanServer(project, matcher, transport, options = {}) {
128
+ if (!project.syncScope) throw new Error('A saved project is required for a server scan')
129
+ const root = validateRemotePath(project.remotePath)
130
+ const remoteInventory = new Map()
131
+ const folders = ['']
132
+ for (let index = 0; index < folders.length; index++) {
133
+ const folder = folders[index]
134
+ const remote = path.posix.join(root, folder)
135
+ options.onProgress?.({ phase: 'inventory', item: { rel: remote }, index, total: folders.length })
136
+ for (const entry of await transport.list(remote)) {
137
+ if (!entry.name || entry.name === '.' || entry.name === '..' || /[\\/\x00-\x1f\x7f]/.test(entry.name)) continue
138
+ const rel = folder ? folder + '/' + entry.name : entry.name
139
+ const isDir = entry.type === 'dir'
140
+ if (entry.type !== 'file' && !isDir) continue
141
+ remoteInventory.set(rel, { isDir, size: entry.size })
142
+ if (isDir && matcher.shouldWatch(path.join(project.localPath, ...rel.split('/')))) folders.push(rel)
143
+ }
144
+ }
145
+ const plan = await buildPlan(project, matcher, transport, { ...options, force: false, remoteInventory })
146
+ const { replaceScannedFiles } = await import('./core/syncState.js')
147
+ replaceScannedFiles(project.syncScope, plan.matches)
148
+ return plan
108
149
  }
109
150
 
110
151
  /**
111
152
  * Upload one file atomically: write to <name>.devsync-uploading, then rename.
112
153
  * Without this, PHP can parse a half-written file and 500 mid-transfer.
113
154
  */
114
- export async function uploadOne(transport, item) {
115
- if (transport.transaction) return transport.transaction((raw) => uploadAttempt(raw, item))
155
+ export async function uploadOne(transport, item, options) {
156
+ if (transport.transaction) return transport.transaction((raw) => uploadAttempt(raw, item), options)
116
157
  return uploadAttempt(transport, item)
117
158
  }
118
159
 
@@ -131,7 +172,19 @@ async function uploadAttempt(transport, item) {
131
172
  }
132
173
  }
133
174
 
134
- export async function runPlan(plan, transport, { dryRun = false, onProgress, onFileResult } = {}) {
175
+ export async function runPlan(plan, transport, { dryRun = false, onProgress, onFileResult, createWorker, concurrency = 3, retries = 2, retryDelayMs = 250, onRetry, onWorkerState } = {}) {
176
+ if (!dryRun && createWorker) {
177
+ return runUploadQueue(plan.uploads, {
178
+ transport, createWorker, concurrency, retries, retryDelayMs, onProgress, onFileResult, onRetry, onWorkerState,
179
+ upload: (client, item) => uploadOne(client, item, { retries: 0 }),
180
+ succeeded: async item => {
181
+ if (plan.syncScope) {
182
+ const { recordUploadedFile } = await import('./core/syncState.js')
183
+ recordUploadedFile(plan.syncScope, item)
184
+ }
185
+ }
186
+ })
187
+ }
135
188
  const done = []
136
189
  const failed = []
137
190
  let bytes = 0
@@ -147,6 +200,10 @@ export async function runPlan(plan, transport, { dryRun = false, onProgress, onF
147
200
  }
148
201
  try {
149
202
  await uploadOne(transport, item)
203
+ if (plan.syncScope) {
204
+ const { recordUploadedFile } = await import('./core/syncState.js')
205
+ recordUploadedFile(plan.syncScope, item)
206
+ }
150
207
  done.push({ rel: item.rel, bytes: item.bytes })
151
208
  bytes += item.bytes
152
209
  if (onFileResult) onFileResult({ item, index: i, total: plan.uploads.length, success: true, dryRun: false, bytes })
package/src/watcher.js CHANGED
@@ -10,6 +10,9 @@ export class ProjectWatcher {
10
10
  this.matcher = matcher
11
11
  this.transport = transport
12
12
  this.debounceMs = opts.debounceMs ?? 400
13
+ this.createWorker = opts.createWorker
14
+ this.concurrency = opts.concurrency ?? 3
15
+ this.onWorkerState = opts.onWorkerState
13
16
  this.dryRun = opts.dryRun ?? false
14
17
  this.onEvent = opts.onEvent ?? (() => {})
15
18
  this.onSettle = opts.onSettle ?? (() => {})
@@ -84,7 +87,9 @@ export class ProjectWatcher {
84
87
  const plan = await buildPlan(this.project, this.matcher, this.transport, { onProgress: this.onProgress })
85
88
  this.onPlan(plan)
86
89
  const result = await runPlan(plan, this.transport, {
87
- dryRun: this.dryRun, onProgress: this.onProgress, onFileResult: this.onFileResult
90
+ dryRun: this.dryRun, createWorker: this.createWorker, concurrency: this.concurrency,
91
+ onProgress: this.onProgress, onFileResult: this.onFileResult, onWorkerState: this.onWorkerState,
92
+ onRetry: event => this.onEvent({ type: 'retry', message: `Queue ${event.workerId}: ${event.connectionFailure ? 'connection unavailable' : event.item.rel + ' interrupted'}; returned to the pending queue.` })
88
93
  })
89
94
  this.stats.cycles++
90
95
  if (!this.dryRun) this.stats.uploaded += result.done.length
@@ -119,4 +124,4 @@ export class ProjectWatcher {
119
124
  if (this.watcher) await this.watcher.close()
120
125
  await this.cycle
121
126
  }
122
- }
127
+ }