shared-hosting-sync 0.1.5 → 0.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +7 -0
- package/package.json +1 -1
- package/src/core/syncState.js +12 -4
- package/src/transport/ftp.js +6 -10
- package/src/transport/sftp.js +4 -1
- package/src/uploader.js +21 -22
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,12 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.1.6 — 18 November 2026
|
|
4
|
+
|
|
5
|
+
- Reuse destination folder setup across each upload connection.
|
|
6
|
+
- Avoid an extra server request for every successful file by using unique temporary
|
|
7
|
+
names and renaming only after transfer.
|
|
8
|
+
- Reduce repeated local path and SQLite statement preparation during large syncs.
|
|
9
|
+
|
|
3
10
|
## 0.1.5 — 18 November 2026
|
|
4
11
|
|
|
5
12
|
- List server folders before comparing local files during an initial scan, avoiding
|
package/package.json
CHANGED
package/src/core/syncState.js
CHANGED
|
@@ -27,11 +27,19 @@ export function syncStatus(scope) {
|
|
|
27
27
|
return { trackedFiles: count, scannedAt: target?.scanned_at || null, ready: !!target }
|
|
28
28
|
}
|
|
29
29
|
|
|
30
|
+
const statements = new WeakMap()
|
|
31
|
+
function prepared(db, sql) {
|
|
32
|
+
let cache = statements.get(db)
|
|
33
|
+
if (!cache) statements.set(db, cache = new Map())
|
|
34
|
+
if (!cache.has(sql)) cache.set(sql, db.prepare(sql))
|
|
35
|
+
return cache.get(sql)
|
|
36
|
+
}
|
|
37
|
+
|
|
30
38
|
function ensureTarget(db, scope) {
|
|
31
|
-
db
|
|
39
|
+
prepared(db, 'INSERT OR IGNORE INTO sync_targets (project_id, target_key) VALUES (?, ?)').run(scope.projectId, scope.targetKey)
|
|
32
40
|
}
|
|
33
41
|
function writeFile(db, scope, rel, value) {
|
|
34
|
-
db
|
|
42
|
+
prepared(db, `INSERT INTO sync_files (project_id, target_key, relative_path, size, mtime_ms, ctime_ms, synced_at)
|
|
35
43
|
VALUES (?, ?, ?, ?, ?, ?, ?) ON CONFLICT(project_id, target_key, relative_path) DO UPDATE SET
|
|
36
44
|
size=excluded.size, mtime_ms=excluded.mtime_ms, ctime_ms=excluded.ctime_ms, synced_at=excluded.synced_at`)
|
|
37
45
|
.run(scope.projectId, scope.targetKey, rel, value.size, value.mtimeMs, value.ctimeMs, new Date().toISOString())
|
|
@@ -41,7 +49,7 @@ function writeFile(db, scope, rel, value) {
|
|
|
41
49
|
export function recordUploadedFile(scope, item) {
|
|
42
50
|
const db = openDb()
|
|
43
51
|
if (!sameFile(item.fingerprint, currentFingerprint(item.abs))) {
|
|
44
|
-
db
|
|
52
|
+
prepared(db, 'DELETE FROM sync_files WHERE project_id = ? AND target_key = ? AND relative_path = ?').run(scope.projectId, scope.targetKey, item.rel)
|
|
45
53
|
return
|
|
46
54
|
}
|
|
47
55
|
ensureTarget(db, scope)
|
|
@@ -54,7 +62,7 @@ export function replaceScannedFiles(scope, matches) {
|
|
|
54
62
|
db.exec('BEGIN IMMEDIATE')
|
|
55
63
|
try {
|
|
56
64
|
ensureTarget(db, scope)
|
|
57
|
-
db
|
|
65
|
+
prepared(db, 'DELETE FROM sync_files WHERE project_id = ? AND target_key = ?').run(scope.projectId, scope.targetKey)
|
|
58
66
|
for (const item of matches) {
|
|
59
67
|
if (sameFile(item.fingerprint, currentFingerprint(item.abs))) writeFile(db, scope, item.rel, item.fingerprint)
|
|
60
68
|
}
|
package/src/transport/ftp.js
CHANGED
|
@@ -9,6 +9,7 @@ import { hashSink } from '../checksum.js'
|
|
|
9
9
|
export class FtpTransport {
|
|
10
10
|
constructor(creds) {
|
|
11
11
|
this.creds = creds
|
|
12
|
+
this.preparedDirs = new Set()
|
|
12
13
|
this.client = null
|
|
13
14
|
}
|
|
14
15
|
|
|
@@ -27,6 +28,7 @@ export class FtpTransport {
|
|
|
27
28
|
c.close()
|
|
28
29
|
throw err
|
|
29
30
|
}
|
|
31
|
+
this.preparedDirs.clear()
|
|
30
32
|
this.client = c
|
|
31
33
|
return this
|
|
32
34
|
}
|
|
@@ -42,16 +44,10 @@ export class FtpTransport {
|
|
|
42
44
|
}
|
|
43
45
|
|
|
44
46
|
async ensureDir(remotePath) {
|
|
45
|
-
if (!remotePath) return
|
|
46
|
-
//
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
let acc = remotePath.startsWith('/') ? '' : '.'
|
|
50
|
-
for (const part of parts) {
|
|
51
|
-
acc = acc === '' || acc === '.' ? `${acc}/${part}` : `${acc}/${part}`
|
|
52
|
-
const current = acc.startsWith('/') ? acc : `/${acc}`
|
|
53
|
-
await this.client.ensureDir(current)
|
|
54
|
-
}
|
|
47
|
+
if (!remotePath || this.preparedDirs.has(remotePath)) return
|
|
48
|
+
// basic-ftp creates intermediate directories itself. Uploads use absolute paths.
|
|
49
|
+
await this.client.ensureDir(remotePath)
|
|
50
|
+
this.preparedDirs.add(remotePath)
|
|
55
51
|
}
|
|
56
52
|
|
|
57
53
|
async exists(remotePath) {
|
package/src/transport/sftp.js
CHANGED
|
@@ -7,6 +7,7 @@ import { finished } from 'node:stream/promises'
|
|
|
7
7
|
export class SftpTransport {
|
|
8
8
|
constructor(creds) {
|
|
9
9
|
this.creds = creds
|
|
10
|
+
this.preparedDirs = new Set()
|
|
10
11
|
this.client = null
|
|
11
12
|
}
|
|
12
13
|
|
|
@@ -30,6 +31,7 @@ export class SftpTransport {
|
|
|
30
31
|
await c.end().catch(() => {})
|
|
31
32
|
throw err
|
|
32
33
|
}
|
|
34
|
+
this.preparedDirs.clear()
|
|
33
35
|
this.client = c
|
|
34
36
|
return this
|
|
35
37
|
}
|
|
@@ -47,7 +49,7 @@ export class SftpTransport {
|
|
|
47
49
|
}
|
|
48
50
|
|
|
49
51
|
async ensureDir(remotePath) {
|
|
50
|
-
if (!remotePath) return
|
|
52
|
+
if (!remotePath || this.preparedDirs.has(remotePath)) return
|
|
51
53
|
const type = await this.client.exists(remotePath)
|
|
52
54
|
if (type && type !== 'd') throw new Error(`Remote directory path is occupied by a file: ${remotePath}`)
|
|
53
55
|
if (!type) {
|
|
@@ -57,6 +59,7 @@ export class SftpTransport {
|
|
|
57
59
|
if (await this.client.exists(remotePath) !== 'd') throw error
|
|
58
60
|
}
|
|
59
61
|
}
|
|
62
|
+
this.preparedDirs.add(remotePath)
|
|
60
63
|
}
|
|
61
64
|
|
|
62
65
|
async exists(remotePath) {
|
package/src/uploader.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import fs from 'node:fs'
|
|
2
|
+
import { randomUUID } from 'node:crypto'
|
|
2
3
|
import { runUploadQueue } from './uploadQueue.js'
|
|
3
4
|
import path from 'node:path'
|
|
4
5
|
import { validateRemotePath, relativeFile } from './paths.js'
|
|
@@ -7,6 +8,7 @@ import { fingerprint, sameFile } from './core/fileFingerprint.js'
|
|
|
7
8
|
import { isTransient } from './transport/reliable.js'
|
|
8
9
|
|
|
9
10
|
const TMP_SUFFIX = '.devsync-uploading'
|
|
11
|
+
const temporaryPaths = new WeakMap()
|
|
10
12
|
|
|
11
13
|
/** Local Windows path -> remote POSIX path, relative to the project root. */
|
|
12
14
|
export function toRemotePath(projectRoot, remoteRoot, localAbsPath) {
|
|
@@ -62,8 +64,9 @@ export function collectFiles(projectRoot, matcher, { includeIgnored = false, onE
|
|
|
62
64
|
*/
|
|
63
65
|
export async function buildPlan(project, matcher, transport, { force = false, remoteInventory, onProgress } = {}) {
|
|
64
66
|
const { localPath, remotePath } = project
|
|
65
|
-
validateRemotePath(remotePath)
|
|
66
|
-
|
|
67
|
+
const root = validateRemotePath(remotePath)
|
|
68
|
+
const realRoot = fs.realpathSync(localPath)
|
|
69
|
+
if (!fs.statSync(realRoot).isDirectory()) throw new Error('Local source must be a directory')
|
|
67
70
|
const excluded = []
|
|
68
71
|
let excludedCount = 0
|
|
69
72
|
const files = collectFiles(localPath, matcher, { onExcluded: (entry) => {
|
|
@@ -78,11 +81,11 @@ export async function buildPlan(project, matcher, transport, { force = false, re
|
|
|
78
81
|
let comparedBytes = 0
|
|
79
82
|
|
|
80
83
|
for (const [index, abs] of files.entries()) {
|
|
81
|
-
const rel =
|
|
82
|
-
const remote =
|
|
84
|
+
const rel = relativeFile(localPath, abs)
|
|
85
|
+
const remote = `${root === '/' ? '' : root}/${rel}`
|
|
83
86
|
let localStat
|
|
84
87
|
try {
|
|
85
|
-
relativeFile(
|
|
88
|
+
relativeFile(realRoot, fs.realpathSync(abs))
|
|
86
89
|
localStat = fs.statSync(abs)
|
|
87
90
|
} catch (err) {
|
|
88
91
|
if (err.code === 'ENOENT') continue
|
|
@@ -92,7 +95,6 @@ export async function buildPlan(project, matcher, transport, { force = false, re
|
|
|
92
95
|
if (onProgress) onProgress({ phase: 'planning', item: { rel, remote, bytes: localStat.size }, index, total: files.length })
|
|
93
96
|
|
|
94
97
|
const localFingerprint = fingerprint(localStat)
|
|
95
|
-
const item = { abs, rel, remote, bytes: localStat.size, dirs: ancestorsOf(remote), fingerprint: localFingerprint }
|
|
96
98
|
if (saved) {
|
|
97
99
|
if (!force && sameFile(saved.get(rel), localFingerprint)) {
|
|
98
100
|
skipped.push({ rel, bytes: localStat.size, reason: 'unchanged since last successful sync' })
|
|
@@ -104,7 +106,7 @@ export async function buildPlan(project, matcher, transport, { force = false, re
|
|
|
104
106
|
if (!force && remoteStat && remoteStat.size === localStat.size &&
|
|
105
107
|
await transport.checksum(remote) === await hashFile(abs)) {
|
|
106
108
|
skipped.push({ rel, bytes: localStat.size, reason: 'same content' })
|
|
107
|
-
matches.push(
|
|
109
|
+
matches.push({ abs, rel, fingerprint: localFingerprint })
|
|
108
110
|
continue
|
|
109
111
|
}
|
|
110
112
|
}
|
|
@@ -149,7 +151,7 @@ export async function scanServer(project, matcher, transport, options = {}) {
|
|
|
149
151
|
}
|
|
150
152
|
|
|
151
153
|
/**
|
|
152
|
-
* Upload one file atomically: write to
|
|
154
|
+
* Upload one file atomically: write to a unique temporary file, then rename.
|
|
153
155
|
* Without this, PHP can parse a half-written file and 500 mid-transfer.
|
|
154
156
|
*/
|
|
155
157
|
export async function uploadOne(transport, item, options) {
|
|
@@ -158,12 +160,15 @@ export async function uploadOne(transport, item, options) {
|
|
|
158
160
|
}
|
|
159
161
|
|
|
160
162
|
async function uploadAttempt(transport, item) {
|
|
161
|
-
|
|
162
|
-
|
|
163
|
+
const parent = item.dirs.at(-1)
|
|
164
|
+
if (parent) await transport.ensureDir(parent)
|
|
165
|
+
// Reuse this file’s temporary name on retry so a leftover partial file is replaced.
|
|
166
|
+
let tmp = temporaryPaths.get(item)
|
|
167
|
+
if (!tmp) {
|
|
168
|
+
tmp = path.posix.join(path.posix.dirname(item.remote), randomUUID() + TMP_SUFFIX)
|
|
169
|
+
temporaryPaths.set(item, tmp)
|
|
163
170
|
}
|
|
164
|
-
const tmp = item.remote + TMP_SUFFIX
|
|
165
171
|
try {
|
|
166
|
-
await transport.delete(tmp).catch(() => {})
|
|
167
172
|
await transport.put(item.abs, tmp)
|
|
168
173
|
await transport.rename(tmp, item.remote)
|
|
169
174
|
} catch (err) {
|
|
@@ -173,16 +178,13 @@ async function uploadAttempt(transport, item) {
|
|
|
173
178
|
}
|
|
174
179
|
|
|
175
180
|
export async function runPlan(plan, transport, { dryRun = false, onProgress, onFileResult, createWorker, concurrency = 3, retries = 2, retryDelayMs = 250, onRetry, onWorkerState } = {}) {
|
|
181
|
+
const record = !dryRun && plan.syncScope && plan.uploads.length
|
|
182
|
+
? (await import('./core/syncState.js')).recordUploadedFile : null
|
|
176
183
|
if (!dryRun && createWorker) {
|
|
177
184
|
return runUploadQueue(plan.uploads, {
|
|
178
185
|
transport, createWorker, concurrency, retries, retryDelayMs, onProgress, onFileResult, onRetry, onWorkerState,
|
|
179
186
|
upload: (client, item) => uploadOne(client, item, { retries: 0 }),
|
|
180
|
-
succeeded:
|
|
181
|
-
if (plan.syncScope) {
|
|
182
|
-
const { recordUploadedFile } = await import('./core/syncState.js')
|
|
183
|
-
recordUploadedFile(plan.syncScope, item)
|
|
184
|
-
}
|
|
185
|
-
}
|
|
187
|
+
succeeded: item => record?.(plan.syncScope, item)
|
|
186
188
|
})
|
|
187
189
|
}
|
|
188
190
|
const done = []
|
|
@@ -200,10 +202,7 @@ export async function runPlan(plan, transport, { dryRun = false, onProgress, onF
|
|
|
200
202
|
}
|
|
201
203
|
try {
|
|
202
204
|
await uploadOne(transport, item)
|
|
203
|
-
|
|
204
|
-
const { recordUploadedFile } = await import('./core/syncState.js')
|
|
205
|
-
recordUploadedFile(plan.syncScope, item)
|
|
206
|
-
}
|
|
205
|
+
record?.(plan.syncScope, item)
|
|
207
206
|
done.push({ rel: item.rel, bytes: item.bytes })
|
|
208
207
|
bytes += item.bytes
|
|
209
208
|
if (onFileResult) onFileResult({ item, index: i, total: plan.uploads.length, success: true, dryRun: false, bytes })
|