phio 0.3.4 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +141 -0
- package/README.md +44 -35
- package/package.json +32 -22
- package/src/cli.ts +9 -4
- package/src/commands/DevCommand.ts +18 -6
- package/src/commands/InfoCommand.ts +27 -4
- package/src/commands/LinkCommand.ts +1 -1
- package/src/commands/LogsCommand.ts +14 -11
- package/src/commands/SftpCommand.ts +99 -0
- package/src/lib/constants.ts +2 -0
- package/src/lib/defaultInstanceId.ts +58 -36
- package/src/lib/deployKey.ts +222 -0
- package/src/lib/fetchEventSource.ts +3 -0
- package/src/lib/getClient.ts +30 -2
- package/src/lib/phioRoot.ts +67 -0
- package/src/lib/sftpConnection.ts +33 -0
- package/src/lib/sshPublicKey.ts +128 -0
- package/vendor/ftp-deploy/HashDiff.ts +122 -0
- package/vendor/ftp-deploy/LICENSE +21 -0
- package/vendor/ftp-deploy/README.md +3 -0
- package/vendor/ftp-deploy/deploy.ts +226 -0
- package/vendor/ftp-deploy/errorHandling.ts +67 -0
- package/vendor/ftp-deploy/localFiles.ts +47 -0
- package/vendor/ftp-deploy/module.ts +28 -0
- package/vendor/ftp-deploy/sftpDeploy.ts +233 -0
- package/vendor/ftp-deploy/sftpSyncProvider.ts +188 -0
- package/vendor/ftp-deploy/sshPrivateKey.ts +66 -0
- package/vendor/ftp-deploy/syncProvider.ts +189 -0
- package/vendor/ftp-deploy/types.ts +226 -0
- package/vendor/ftp-deploy/utilities.ts +217 -0
- package/src/commands/WhoAmICommand.ts +0 -15
- package/src/global.d.ts +0 -3
|
@@ -0,0 +1,233 @@
|
|
|
1
|
+
import fs from "fs";
|
|
2
|
+
import SftpClient from "ssh2-sftp-client";
|
|
3
|
+
import { IFileList, IDiff, syncFileDescription, currentSyncFileVersion, IFtpDeployArgumentsWithDefaults } from "./types";
|
|
4
|
+
import { HashDiff } from "./HashDiff";
|
|
5
|
+
import { ILogger, retryRequest, ITimings, applyExcludeFilter, formatNumber } from "./utilities";
|
|
6
|
+
import prettyBytes from "pretty-bytes";
|
|
7
|
+
import { prettyError } from "./errorHandling";
|
|
8
|
+
import { ensureSftpDir, SFTPSyncProvider } from "./sftpSyncProvider";
|
|
9
|
+
import { getLocalFiles } from "./localFiles";
|
|
10
|
+
import { readPrivateKeyForSsh2 } from "./sshPrivateKey";
|
|
11
|
+
|
|
12
|
+
async function downloadFileList(client: SftpClient, logger: ILogger, path: string): Promise<IFileList> {
|
|
13
|
+
const tempFileNameHack = ".ftp-deploy-sync-server-state-buffer-file---delete.json";
|
|
14
|
+
|
|
15
|
+
await retryRequest(logger, async () => await client.fastGet(path, tempFileNameHack));
|
|
16
|
+
|
|
17
|
+
const fileAsString = fs.readFileSync(tempFileNameHack, { encoding: "utf-8" });
|
|
18
|
+
const fileAsObject = JSON.parse(fileAsString) as IFileList;
|
|
19
|
+
|
|
20
|
+
fs.unlinkSync(tempFileNameHack);
|
|
21
|
+
|
|
22
|
+
return fileAsObject;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
function createLocalState(localFiles: IFileList, logger: ILogger, args: IFtpDeployArgumentsWithDefaults): void {
|
|
26
|
+
logger.verbose(`Creating local state at ${args["local-dir"]}${args["state-name"]}`);
|
|
27
|
+
fs.writeFileSync(`${args["local-dir"]}${args["state-name"]}`, JSON.stringify(localFiles, undefined, 4), {
|
|
28
|
+
encoding: "utf8",
|
|
29
|
+
});
|
|
30
|
+
logger.verbose("Local state created");
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
async function connect(client: SftpClient, args: IFtpDeployArgumentsWithDefaults, logger: ILogger) {
|
|
34
|
+
if (!args["private-key-path"]) {
|
|
35
|
+
throw new Error(`SFTP deploy requires "private-key-path"`);
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
const privateKey = readPrivateKeyForSsh2(args["private-key-path"]);
|
|
39
|
+
|
|
40
|
+
try {
|
|
41
|
+
await client.connect({
|
|
42
|
+
host: args.server,
|
|
43
|
+
port: args.port,
|
|
44
|
+
username: args.username,
|
|
45
|
+
privateKey,
|
|
46
|
+
readyTimeout: args.timeout,
|
|
47
|
+
});
|
|
48
|
+
} catch (error) {
|
|
49
|
+
logger.all("Failed to connect over SFTP. Check host, port, username, and deploy key registration.");
|
|
50
|
+
throw error;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
const serverDir = args["server-dir"].replace(/\/$/, "") || ".";
|
|
54
|
+
await retryRequest(logger, async () => await client.realPath(serverDir));
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
async function getServerFiles(
|
|
58
|
+
client: SftpClient,
|
|
59
|
+
logger: ILogger,
|
|
60
|
+
timings: ITimings,
|
|
61
|
+
args: IFtpDeployArgumentsWithDefaults
|
|
62
|
+
): Promise<IFileList> {
|
|
63
|
+
try {
|
|
64
|
+
await ensureSftpDir(client, logger, timings, ".");
|
|
65
|
+
|
|
66
|
+
if (args["dangerous-clean-slate"]) {
|
|
67
|
+
logger.all(`----------------------------------------------------------------`);
|
|
68
|
+
logger.all(
|
|
69
|
+
"🗑️ Removing all files on the server because 'dangerous-clean-slate' was set, this will make the deployment very slow..."
|
|
70
|
+
);
|
|
71
|
+
if (args["dry-run"] === false) {
|
|
72
|
+
const entries = await client.list(".");
|
|
73
|
+
for (const entry of entries) {
|
|
74
|
+
if (entry.name === "." || entry.name === "..") continue;
|
|
75
|
+
if (entry.type === "d") {
|
|
76
|
+
await client.rmdir(entry.name, true);
|
|
77
|
+
} else {
|
|
78
|
+
await client.delete(entry.name);
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
logger.all("Clear complete");
|
|
83
|
+
|
|
84
|
+
throw new Error("dangerous-clean-slate was run");
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
const serverFiles = await downloadFileList(client, logger, args["state-name"]);
|
|
88
|
+
logger.all(`----------------------------------------------------------------`);
|
|
89
|
+
logger.all(
|
|
90
|
+
`Last published on 📅 ${new Date(serverFiles.generatedTime).toLocaleDateString(undefined, {
|
|
91
|
+
weekday: "long",
|
|
92
|
+
year: "numeric",
|
|
93
|
+
month: "long",
|
|
94
|
+
day: "numeric",
|
|
95
|
+
hour: "numeric",
|
|
96
|
+
minute: "numeric",
|
|
97
|
+
})}`
|
|
98
|
+
);
|
|
99
|
+
|
|
100
|
+
if (args.exclude.length > 0) {
|
|
101
|
+
const filteredData = serverFiles.data.filter((item) =>
|
|
102
|
+
applyExcludeFilter({ path: item.name, isDirectory: () => item.type === "folder" }, args.exclude)
|
|
103
|
+
);
|
|
104
|
+
serverFiles.data = filteredData;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
return serverFiles;
|
|
108
|
+
} catch (error) {
|
|
109
|
+
logger.all(`----------------------------------------------------------------`);
|
|
110
|
+
logger.all(`No file exists on the server "${args["server-dir"] + args["state-name"]}" - this must be your first publish! 🎉`);
|
|
111
|
+
logger.all(`The first publish will take a while... but once the initial sync is done only differences are published!`);
|
|
112
|
+
logger.all(`If you get this message and its NOT your first publish, something is wrong.`);
|
|
113
|
+
|
|
114
|
+
return {
|
|
115
|
+
description: syncFileDescription,
|
|
116
|
+
version: currentSyncFileVersion,
|
|
117
|
+
generatedTime: new Date().getTime(),
|
|
118
|
+
data: [],
|
|
119
|
+
};
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
export async function deploySftp(args: IFtpDeployArgumentsWithDefaults, logger: ILogger, timings: ITimings): Promise<void> {
|
|
124
|
+
timings.start("total");
|
|
125
|
+
|
|
126
|
+
logger.all(`----------------------------------------------------------------`);
|
|
127
|
+
logger.all(`🚀 Thanks for using ftp-deploy. Let's deploy some stuff! `);
|
|
128
|
+
logger.all(`----------------------------------------------------------------`);
|
|
129
|
+
logger.verbose(`Using the following include filters: ${JSON.stringify(args.include)}`);
|
|
130
|
+
logger.verbose(`Using the following excludes filters: ${JSON.stringify(args.exclude)}`);
|
|
131
|
+
|
|
132
|
+
timings.start("hash");
|
|
133
|
+
const localFiles = await getLocalFiles(args, logger);
|
|
134
|
+
timings.stop("hash");
|
|
135
|
+
|
|
136
|
+
createLocalState(localFiles, logger, args);
|
|
137
|
+
|
|
138
|
+
const client = new SftpClient();
|
|
139
|
+
|
|
140
|
+
let totalBytesUploaded = 0;
|
|
141
|
+
try {
|
|
142
|
+
timings.start("connecting");
|
|
143
|
+
await connect(client, args, logger);
|
|
144
|
+
timings.stop("connecting");
|
|
145
|
+
|
|
146
|
+
const serverFiles = await getServerFiles(client, logger, timings, args);
|
|
147
|
+
|
|
148
|
+
timings.start("logging");
|
|
149
|
+
const diffTool: IDiff = new HashDiff();
|
|
150
|
+
|
|
151
|
+
logger.standard(`----------------------------------------------------------------`);
|
|
152
|
+
logger.standard(`Local Files:\t${formatNumber(localFiles.data.length)}`);
|
|
153
|
+
logger.standard(`Server Files:\t${formatNumber(serverFiles.data.length)}`);
|
|
154
|
+
logger.standard(`----------------------------------------------------------------`);
|
|
155
|
+
logger.standard(`Calculating differences between client & server`);
|
|
156
|
+
logger.standard(`----------------------------------------------------------------`);
|
|
157
|
+
logger.verbose(`Local files:`, JSON.stringify(localFiles, null, 2));
|
|
158
|
+
logger.verbose(`Server files:`, JSON.stringify(serverFiles, null, 2));
|
|
159
|
+
|
|
160
|
+
const diffs = diffTool.getDiffs(localFiles, serverFiles);
|
|
161
|
+
|
|
162
|
+
diffs.upload
|
|
163
|
+
.filter((itemUpload) => itemUpload.type === "folder")
|
|
164
|
+
.map((itemUpload) => {
|
|
165
|
+
logger.standard(`📁 Create: ${itemUpload.name}`);
|
|
166
|
+
});
|
|
167
|
+
|
|
168
|
+
diffs.upload
|
|
169
|
+
.filter((itemUpload) => itemUpload.type === "file")
|
|
170
|
+
.map((itemUpload) => {
|
|
171
|
+
logger.standard(`📄 Upload: ${itemUpload.name}`);
|
|
172
|
+
});
|
|
173
|
+
|
|
174
|
+
diffs.replace.map((itemReplace) => {
|
|
175
|
+
logger.standard(`🔁 File replace: ${itemReplace.name}`);
|
|
176
|
+
});
|
|
177
|
+
|
|
178
|
+
diffs.delete
|
|
179
|
+
.filter((itemUpload) => itemUpload.type === "file")
|
|
180
|
+
.map((itemDelete) => {
|
|
181
|
+
logger.standard(`📄 Delete: ${itemDelete.name} `);
|
|
182
|
+
});
|
|
183
|
+
|
|
184
|
+
diffs.delete
|
|
185
|
+
.filter((itemUpload) => itemUpload.type === "folder")
|
|
186
|
+
.map((itemDelete) => {
|
|
187
|
+
logger.standard(`📁 Delete: ${itemDelete.name} `);
|
|
188
|
+
});
|
|
189
|
+
|
|
190
|
+
diffs.same.map((itemSame) => {
|
|
191
|
+
if (itemSame.type === "file") {
|
|
192
|
+
logger.standard(`⚖️ File content is the same, doing nothing: ${itemSame.name}`);
|
|
193
|
+
}
|
|
194
|
+
});
|
|
195
|
+
timings.stop("logging");
|
|
196
|
+
|
|
197
|
+
totalBytesUploaded = diffs.sizeUpload + diffs.sizeReplace;
|
|
198
|
+
|
|
199
|
+
timings.start("upload");
|
|
200
|
+
try {
|
|
201
|
+
const syncProvider = new SFTPSyncProvider(
|
|
202
|
+
client,
|
|
203
|
+
logger,
|
|
204
|
+
timings,
|
|
205
|
+
args["local-dir"],
|
|
206
|
+
args["server-dir"],
|
|
207
|
+
args["state-name"],
|
|
208
|
+
args["dry-run"]
|
|
209
|
+
);
|
|
210
|
+
await syncProvider.syncLocalToServer(diffs);
|
|
211
|
+
} finally {
|
|
212
|
+
timings.stop("upload");
|
|
213
|
+
}
|
|
214
|
+
} catch (error) {
|
|
215
|
+
prettyError(logger, args, error);
|
|
216
|
+
throw error;
|
|
217
|
+
} finally {
|
|
218
|
+
await client.end();
|
|
219
|
+
timings.stop("total");
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
const uploadSpeed = prettyBytes(totalBytesUploaded / (timings.getTime("upload") / 1000));
|
|
223
|
+
|
|
224
|
+
logger.all(`----------------------------------------------------------------`);
|
|
225
|
+
logger.all(`Time spent hashing: ${timings.getTimeFormatted("hash")}`);
|
|
226
|
+
logger.all(`Time spent connecting to server: ${timings.getTimeFormatted("connecting")}`);
|
|
227
|
+
logger.all(`Time spent deploying: ${timings.getTimeFormatted("upload")} (${uploadSpeed}/second)`);
|
|
228
|
+
logger.all(` - changing dirs: ${timings.getTimeFormatted("changingDir")}`);
|
|
229
|
+
logger.all(` - logging: ${timings.getTimeFormatted("logging")}`);
|
|
230
|
+
logger.all(`----------------------------------------------------------------`);
|
|
231
|
+
logger.all(`Total time: ${timings.getTimeFormatted("total")}`);
|
|
232
|
+
logger.all(`----------------------------------------------------------------`);
|
|
233
|
+
}
|
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
import prettyBytes from "pretty-bytes";
|
|
2
|
+
import type SftpClient from "ssh2-sftp-client";
|
|
3
|
+
import { DiffResult, IFilePath } from "./types";
|
|
4
|
+
import { ISyncProvider } from "./syncProvider";
|
|
5
|
+
import { ILogger, pluralize, retryRequest, ITimings } from "./utilities";
|
|
6
|
+
|
|
7
|
+
export async function ensureSftpDir(
|
|
8
|
+
client: SftpClient,
|
|
9
|
+
logger: ILogger,
|
|
10
|
+
timings: ITimings,
|
|
11
|
+
folder: string
|
|
12
|
+
): Promise<void> {
|
|
13
|
+
timings.start("changingDir");
|
|
14
|
+
logger.verbose(` ensuring dir ${folder}`);
|
|
15
|
+
|
|
16
|
+
await retryRequest(logger, async () => {
|
|
17
|
+
const target = folder === "." || folder === "" ? "." : folder;
|
|
18
|
+
if (target !== ".") {
|
|
19
|
+
const exists = await client.exists(target);
|
|
20
|
+
if (!exists) {
|
|
21
|
+
await client.mkdir(target, true);
|
|
22
|
+
}
|
|
23
|
+
await client.realPath(target);
|
|
24
|
+
}
|
|
25
|
+
});
|
|
26
|
+
|
|
27
|
+
logger.verbose(` dir ready`);
|
|
28
|
+
timings.stop("changingDir");
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
export class SFTPSyncProvider implements ISyncProvider {
|
|
32
|
+
constructor(
|
|
33
|
+
client: SftpClient,
|
|
34
|
+
logger: ILogger,
|
|
35
|
+
timings: ITimings,
|
|
36
|
+
localPath: string,
|
|
37
|
+
serverPath: string,
|
|
38
|
+
stateName: string,
|
|
39
|
+
dryRun: boolean
|
|
40
|
+
) {
|
|
41
|
+
this.client = client;
|
|
42
|
+
this.logger = logger;
|
|
43
|
+
this.timings = timings;
|
|
44
|
+
this.localPath = localPath;
|
|
45
|
+
this.serverPath = serverPath;
|
|
46
|
+
this.stateName = stateName;
|
|
47
|
+
this.dryRun = dryRun;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
private client: SftpClient;
|
|
51
|
+
private logger: ILogger;
|
|
52
|
+
private timings: ITimings;
|
|
53
|
+
private localPath: string;
|
|
54
|
+
private serverPath: string;
|
|
55
|
+
private dryRun: boolean;
|
|
56
|
+
private stateName: string;
|
|
57
|
+
|
|
58
|
+
private getFileBreadcrumbs(fullPath: string): IFilePath {
|
|
59
|
+
const pathSplit = fullPath.split("/");
|
|
60
|
+
const file = pathSplit?.pop() ?? "";
|
|
61
|
+
const folders = pathSplit.filter((folderName) => folderName != "");
|
|
62
|
+
|
|
63
|
+
return {
|
|
64
|
+
folders: folders.length === 0 ? null : folders,
|
|
65
|
+
file: file === "" ? null : file,
|
|
66
|
+
};
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
private async upDir(dirCount: number | null | undefined): Promise<void> {
|
|
70
|
+
if (typeof dirCount !== "number") {
|
|
71
|
+
return;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
for (let i = 0; i < dirCount; i++) {
|
|
75
|
+
await retryRequest(this.logger, async () => await this.client.realPath(".."));
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
async createFolder(folderPath: string) {
|
|
80
|
+
this.logger.all(`creating folder "${folderPath + "/"}"`);
|
|
81
|
+
|
|
82
|
+
if (this.dryRun === true) {
|
|
83
|
+
return;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
const path = this.getFileBreadcrumbs(folderPath + "/");
|
|
87
|
+
|
|
88
|
+
if (path.folders === null) {
|
|
89
|
+
this.logger.verbose(` no need to change dir`);
|
|
90
|
+
} else {
|
|
91
|
+
await ensureSftpDir(this.client, this.logger, this.timings, path.folders.join("/"));
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
await this.upDir(path.folders?.length);
|
|
95
|
+
|
|
96
|
+
this.logger.verbose(` completed`);
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
async removeFile(filePath: string) {
|
|
100
|
+
this.logger.all(`removing "${filePath}"`);
|
|
101
|
+
|
|
102
|
+
if (this.dryRun === false) {
|
|
103
|
+
try {
|
|
104
|
+
await retryRequest(this.logger, async () => await this.client.delete(filePath));
|
|
105
|
+
} catch (e: any) {
|
|
106
|
+
const message = `${e?.message ?? e}`.toLowerCase();
|
|
107
|
+
if (message.includes("no such file") || message.includes("not found")) {
|
|
108
|
+
this.logger.standard("File not found or you don't have access to the file - skipping...");
|
|
109
|
+
} else {
|
|
110
|
+
throw e;
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
this.logger.verbose(` file removed`);
|
|
115
|
+
this.logger.verbose(` completed`);
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
async removeFolder(folderPath: string) {
|
|
119
|
+
const absoluteFolderPath =
|
|
120
|
+
"/" +
|
|
121
|
+
(this.serverPath.startsWith("./") ? this.serverPath.replace("./", "") : this.serverPath) +
|
|
122
|
+
folderPath;
|
|
123
|
+
this.logger.all(`removing folder "${absoluteFolderPath}"`);
|
|
124
|
+
|
|
125
|
+
if (this.dryRun === false) {
|
|
126
|
+
await retryRequest(this.logger, async () => await this.client.rmdir(folderPath));
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
this.logger.verbose(` completed`);
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
async uploadFile(filePath: string, type: "upload" | "replace" = "upload") {
|
|
133
|
+
const typePresent = type === "upload" ? "uploading" : "replacing";
|
|
134
|
+
const typePast = type === "upload" ? "uploaded" : "replaced";
|
|
135
|
+
this.logger.all(`${typePresent} "${filePath}"`);
|
|
136
|
+
|
|
137
|
+
if (this.dryRun === false) {
|
|
138
|
+
await retryRequest(
|
|
139
|
+
this.logger,
|
|
140
|
+
async () => await this.client.fastPut(this.localPath + filePath, filePath)
|
|
141
|
+
);
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
this.logger.verbose(` file ${typePast}`);
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
async syncLocalToServer(diffs: DiffResult) {
|
|
148
|
+
const totalCount = diffs.delete.length + diffs.upload.length + diffs.replace.length;
|
|
149
|
+
|
|
150
|
+
this.logger.all(`----------------------------------------------------------------`);
|
|
151
|
+
this.logger.all(
|
|
152
|
+
`Making changes to ${totalCount} ${pluralize(totalCount, "file/folder", "files/folders")} to sync server state`
|
|
153
|
+
);
|
|
154
|
+
this.logger.all(
|
|
155
|
+
`Uploading: ${prettyBytes(diffs.sizeUpload)} -- Deleting: ${prettyBytes(diffs.sizeDelete)} -- Replacing: ${prettyBytes(diffs.sizeReplace)}`
|
|
156
|
+
);
|
|
157
|
+
this.logger.all(`----------------------------------------------------------------`);
|
|
158
|
+
|
|
159
|
+
for (const file of diffs.upload.filter((item) => item.type === "folder")) {
|
|
160
|
+
await this.createFolder(file.name);
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
for (const file of diffs.upload.filter((item) => item.type === "file").filter((item) => item.name !== this.stateName)) {
|
|
164
|
+
await this.uploadFile(file.name, "upload");
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
for (const file of diffs.replace.filter((item) => item.type === "file").filter((item) => item.name !== this.stateName)) {
|
|
168
|
+
await this.uploadFile(file.name, "replace");
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
for (const file of diffs.delete.filter((item) => item.type === "file")) {
|
|
172
|
+
await this.removeFile(file.name);
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
for (const file of diffs.delete.filter((item) => item.type === "folder")) {
|
|
176
|
+
await this.removeFolder(file.name);
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
this.logger.all(`----------------------------------------------------------------`);
|
|
180
|
+
this.logger.all(`🎉 Sync complete. Saving current server state to "${this.serverPath + this.stateName}"`);
|
|
181
|
+
if (this.dryRun === false) {
|
|
182
|
+
await retryRequest(
|
|
183
|
+
this.logger,
|
|
184
|
+
async () => await this.client.fastPut(this.localPath + this.stateName, this.stateName)
|
|
185
|
+
);
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
}
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
import { createPrivateKey, randomBytes } from 'crypto'
|
|
2
|
+
import { readFileSync } from 'fs'
|
|
3
|
+
|
|
4
|
+
const writeSshString = (value: Buffer | string) => {
|
|
5
|
+
const buf = Buffer.isBuffer(value) ? value : Buffer.from(value)
|
|
6
|
+
const len = Buffer.alloc(4)
|
|
7
|
+
len.writeUInt32BE(buf.length)
|
|
8
|
+
return Buffer.concat([len, buf])
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
export const isPkcs8PrivateKey = (pem: string) => /-----BEGIN PRIVATE KEY-----/.test(pem)
|
|
12
|
+
|
|
13
|
+
/** ssh2 only accepts OpenSSH / legacy PEM formats, not PKCS#8 Ed25519. */
|
|
14
|
+
export const pkcs8Ed25519ToOpenSshPrivateKeyPem = (pem: string, comment = 'phio') => {
|
|
15
|
+
const key = createPrivateKey(pem)
|
|
16
|
+
if (key.asymmetricKeyType !== 'ed25519') {
|
|
17
|
+
throw new Error('Deploy key must be Ed25519')
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
const jwk = key.export({ format: 'jwk' }) as { d: string; x: string }
|
|
21
|
+
const seed = Buffer.from(jwk.d, 'base64url')
|
|
22
|
+
const pub = Buffer.from(jwk.x, 'base64url')
|
|
23
|
+
const pubWire = Buffer.concat([writeSshString('ssh-ed25519'), writeSshString(pub)])
|
|
24
|
+
|
|
25
|
+
const check = randomBytes(4)
|
|
26
|
+
const privPlain = Buffer.concat([
|
|
27
|
+
check,
|
|
28
|
+
check,
|
|
29
|
+
writeSshString('ssh-ed25519'),
|
|
30
|
+
writeSshString(pub),
|
|
31
|
+
writeSshString(Buffer.concat([seed, pub])),
|
|
32
|
+
writeSshString(comment),
|
|
33
|
+
])
|
|
34
|
+
|
|
35
|
+
const blockSize = 8
|
|
36
|
+
const padLen = blockSize - (privPlain.length % blockSize)
|
|
37
|
+
const padding = Buffer.alloc(padLen)
|
|
38
|
+
for (let i = 0; i < padLen; i++) {
|
|
39
|
+
padding[i] = i + 1
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
const blob = Buffer.concat([
|
|
43
|
+
Buffer.from('openssh-key-v1\0'),
|
|
44
|
+
writeSshString('none'),
|
|
45
|
+
writeSshString('none'),
|
|
46
|
+
writeSshString(''),
|
|
47
|
+
(() => {
|
|
48
|
+
const count = Buffer.alloc(4)
|
|
49
|
+
count.writeUInt32BE(1)
|
|
50
|
+
return count
|
|
51
|
+
})(),
|
|
52
|
+
writeSshString(pubWire),
|
|
53
|
+
writeSshString(Buffer.concat([privPlain, padding])),
|
|
54
|
+
])
|
|
55
|
+
|
|
56
|
+
const b64 = blob.toString('base64').replace(/.{70}/g, '$&\n')
|
|
57
|
+
return `-----BEGIN OPENSSH PRIVATE KEY-----\n${b64}\n-----END OPENSSH PRIVATE KEY-----\n`
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
export const readPrivateKeyForSsh2 = (privateKeyPath: string) => {
|
|
61
|
+
const pem = readFileSync(privateKeyPath, 'utf8')
|
|
62
|
+
if (isPkcs8PrivateKey(pem)) {
|
|
63
|
+
return pkcs8Ed25519ToOpenSshPrivateKeyPem(pem)
|
|
64
|
+
}
|
|
65
|
+
return pem
|
|
66
|
+
}
|
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
import prettyBytes from "pretty-bytes";
|
|
2
|
+
import type * as ftp from "basic-ftp";
|
|
3
|
+
import { DiffResult, ErrorCode, IFilePath } from "./types";
|
|
4
|
+
import { ILogger, pluralize, retryRequest, ITimings } from "./utilities";
|
|
5
|
+
|
|
6
|
+
export async function ensureDir(client: ftp.Client, logger: ILogger, timings: ITimings, folder: string): Promise<void> {
|
|
7
|
+
timings.start("changingDir");
|
|
8
|
+
logger.verbose(` changing dir to ${folder}`);
|
|
9
|
+
|
|
10
|
+
await retryRequest(logger, async () => await client.ensureDir(folder));
|
|
11
|
+
|
|
12
|
+
logger.verbose(` dir changed`);
|
|
13
|
+
timings.stop("changingDir");
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
export interface ISyncProvider {
|
|
17
|
+
createFolder(folderPath: string): Promise<void>;
|
|
18
|
+
removeFile(filePath: string): Promise<void>;
|
|
19
|
+
removeFolder(folderPath: string): Promise<void>;
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* @param file file can include folder(s)
|
|
23
|
+
* Note working dir is modified and NOT reset after upload
|
|
24
|
+
* For now we are going to reset it - but this will be removed for performance
|
|
25
|
+
*/
|
|
26
|
+
uploadFile(filePath: string, type: "upload" | "replace"): Promise<void>;
|
|
27
|
+
|
|
28
|
+
syncLocalToServer(diffs: DiffResult): Promise<void>;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
export class FTPSyncProvider implements ISyncProvider {
|
|
32
|
+
constructor(client: ftp.Client, logger: ILogger, timings: ITimings, localPath: string, serverPath: string, stateName: string, dryRun: boolean) {
|
|
33
|
+
this.client = client;
|
|
34
|
+
this.logger = logger;
|
|
35
|
+
this.timings = timings;
|
|
36
|
+
this.localPath = localPath;
|
|
37
|
+
this.serverPath = serverPath;
|
|
38
|
+
this.stateName = stateName;
|
|
39
|
+
this.dryRun = dryRun;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
private client: ftp.Client;
|
|
43
|
+
private logger: ILogger;
|
|
44
|
+
private timings: ITimings;
|
|
45
|
+
private localPath: string;
|
|
46
|
+
private serverPath: string;
|
|
47
|
+
private dryRun: boolean;
|
|
48
|
+
private stateName: string;
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* Converts a file path (ex: "folder/otherfolder/file.txt") to an array of folder and a file path
|
|
53
|
+
* @param fullPath
|
|
54
|
+
*/
|
|
55
|
+
private getFileBreadcrumbs(fullPath: string): IFilePath {
|
|
56
|
+
// todo see if this regex will work for nonstandard folder names
|
|
57
|
+
// todo what happens if the path is relative to the root dir? (starts with /)
|
|
58
|
+
const pathSplit = fullPath.split("/");
|
|
59
|
+
const file = pathSplit?.pop() ?? ""; // get last item
|
|
60
|
+
const folders = pathSplit.filter(folderName => folderName != "");
|
|
61
|
+
|
|
62
|
+
return {
|
|
63
|
+
folders: folders.length === 0 ? null : folders,
|
|
64
|
+
file: file === "" ? null : file
|
|
65
|
+
};
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* Navigates up {dirCount} number of directories from the current working dir
|
|
70
|
+
*/
|
|
71
|
+
private async upDir(dirCount: number | null | undefined): Promise<void> {
|
|
72
|
+
if (typeof dirCount !== "number") {
|
|
73
|
+
return;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
// navigate back to the starting folder
|
|
77
|
+
for (let i = 0; i < dirCount; i++) {
|
|
78
|
+
await retryRequest(this.logger, async () => await this.client.cdup());
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
async createFolder(folderPath: string) {
|
|
83
|
+
this.logger.all(`creating folder "${folderPath + "/"}"`);
|
|
84
|
+
|
|
85
|
+
if (this.dryRun === true) {
|
|
86
|
+
return;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
const path = this.getFileBreadcrumbs(folderPath + "/");
|
|
90
|
+
|
|
91
|
+
if (path.folders === null) {
|
|
92
|
+
this.logger.verbose(` no need to change dir`);
|
|
93
|
+
}
|
|
94
|
+
else {
|
|
95
|
+
await ensureDir(this.client, this.logger, this.timings, path.folders.join("/"));
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
// navigate back to the root folder
|
|
99
|
+
await this.upDir(path.folders?.length);
|
|
100
|
+
|
|
101
|
+
this.logger.verbose(` completed`);
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
async removeFile(filePath: string) {
|
|
105
|
+
this.logger.all(`removing "${filePath}"`);
|
|
106
|
+
|
|
107
|
+
if (this.dryRun === false) {
|
|
108
|
+
try {
|
|
109
|
+
await retryRequest(this.logger, async () => await this.client.remove(filePath));
|
|
110
|
+
}
|
|
111
|
+
catch (e: any) {
|
|
112
|
+
// this error is common when a file was deleted on the server directly
|
|
113
|
+
if (e.code === ErrorCode.FileNotFoundOrNoAccess) {
|
|
114
|
+
this.logger.standard("File not found or you don't have access to the file - skipping...");
|
|
115
|
+
}
|
|
116
|
+
else {
|
|
117
|
+
throw e;
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
this.logger.verbose(` file removed`);
|
|
122
|
+
|
|
123
|
+
this.logger.verbose(` completed`);
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
async removeFolder(folderPath: string) {
|
|
127
|
+
const absoluteFolderPath = "/" + (this.serverPath.startsWith("./") ? this.serverPath.replace("./", "") : this.serverPath) + folderPath;
|
|
128
|
+
this.logger.all(`removing folder "${absoluteFolderPath}"`);
|
|
129
|
+
|
|
130
|
+
if (this.dryRun === false) {
|
|
131
|
+
await retryRequest(this.logger, async () => await this.client.removeDir(absoluteFolderPath));
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
this.logger.verbose(` completed`);
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
async uploadFile(filePath: string, type: "upload" | "replace" = "upload") {
|
|
138
|
+
const typePresent = type === "upload" ? "uploading" : "replacing";
|
|
139
|
+
const typePast = type === "upload" ? "uploaded" : "replaced";
|
|
140
|
+
this.logger.all(`${typePresent} "${filePath}"`);
|
|
141
|
+
|
|
142
|
+
if (this.dryRun === false) {
|
|
143
|
+
await retryRequest(this.logger, async () => await this.client.uploadFrom(this.localPath + filePath, filePath));
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
this.logger.verbose(` file ${typePast}`);
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
async syncLocalToServer(diffs: DiffResult) {
|
|
150
|
+
const totalCount = diffs.delete.length + diffs.upload.length + diffs.replace.length;
|
|
151
|
+
|
|
152
|
+
this.logger.all(`----------------------------------------------------------------`);
|
|
153
|
+
this.logger.all(`Making changes to ${totalCount} ${pluralize(totalCount, "file/folder", "files/folders")} to sync server state`);
|
|
154
|
+
this.logger.all(`Uploading: ${prettyBytes(diffs.sizeUpload)} -- Deleting: ${prettyBytes(diffs.sizeDelete)} -- Replacing: ${prettyBytes(diffs.sizeReplace)}`);
|
|
155
|
+
this.logger.all(`----------------------------------------------------------------`);
|
|
156
|
+
|
|
157
|
+
// create new folders
|
|
158
|
+
for (const file of diffs.upload.filter(item => item.type === "folder")) {
|
|
159
|
+
await this.createFolder(file.name);
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
// upload new files
|
|
163
|
+
for (const file of diffs.upload.filter(item => item.type === "file").filter(item => item.name !== this.stateName)) {
|
|
164
|
+
await this.uploadFile(file.name, "upload");
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
// replace new files
|
|
168
|
+
for (const file of diffs.replace.filter(item => item.type === "file").filter(item => item.name !== this.stateName)) {
|
|
169
|
+
// note: FTP will replace old files with new files. We run replacements after uploads to limit downtime
|
|
170
|
+
await this.uploadFile(file.name, "replace");
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
// delete old files
|
|
174
|
+
for (const file of diffs.delete.filter(item => item.type === "file")) {
|
|
175
|
+
await this.removeFile(file.name);
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
// delete old folders
|
|
179
|
+
for (const file of diffs.delete.filter(item => item.type === "folder")) {
|
|
180
|
+
await this.removeFolder(file.name);
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
this.logger.all(`----------------------------------------------------------------`);
|
|
184
|
+
this.logger.all(`🎉 Sync complete. Saving current server state to "${this.serverPath + this.stateName}"`);
|
|
185
|
+
if (this.dryRun === false) {
|
|
186
|
+
await retryRequest(this.logger, async () => await this.client.uploadFrom(this.localPath + this.stateName, this.stateName));
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
}
|