phio 0.3.5 → 0.4.0

package/vendor/ftp-deploy/HashDiff.ts

@@ -0,0 +1,122 @@
+import { IDiff, IFileList, Record } from "./types";
+import fs from "fs";
+import crypto from "crypto";
+
+export async function fileHash(filename: string, algorithm: "md5" | "sha1" | "sha256" | "sha512"): Promise<string> {
+    return new Promise((resolve, reject) => {
+        // Algorithm depends on availability of OpenSSL on platform
+        // Another algorithms: "sha1", "md5", "sha256", "sha512" ...
+        let shasum = crypto.createHash(algorithm);
+        try {
+            let s = fs.createReadStream(filename);
+            s.on("data", function (data: any) {
+                shasum.update(data)
+            });
+
+            s.on("error", function (error) {
+                reject(error);
+            });
+
+            // making digest
+            s.on("end", function () {
+                const hash = shasum.digest("hex")
+                return resolve(hash);
+            });
+        }
+        catch (error) {
+            return reject("calc fail");
+        }
+    });
+}
+
+export class HashDiff implements IDiff {
+    getDiffs(localFiles: IFileList, serverFiles: IFileList) {
+        const uploadList: Record[] = [];
+        const deleteList: Record[] = [];
+        const replaceList: Record[] = [];
+
+        const sameList: Record[] = [];
+
+        let sizeUpload = 0;
+        let sizeDelete = 0;
+        let sizeReplace = 0;
+
+        // alphabetize each list based off path
+        const localFilesSorted = localFiles.data.sort((first, second) => first.name.localeCompare(second.name));
+        const serverFilesSorted = serverFiles.data.sort((first, second) => first.name.localeCompare(second.name));
+
+        let localPosition = 0;
+        let serverPosition = 0;
+        while (localPosition + serverPosition < localFilesSorted.length + serverFilesSorted.length) {
+            let localFile: Record | undefined = localFilesSorted[localPosition];
+            let serverFile: Record | undefined = serverFilesSorted[serverPosition];
+
+            let fileNameCompare = 0;
+            if (localFile === undefined) {
+                fileNameCompare = 1;
+            }
+            if (serverFile === undefined) {
+                fileNameCompare = -1;
+            }
+            if (localFile !== undefined && serverFile !== undefined) {
+                fileNameCompare = localFile.name.localeCompare(serverFile.name);
+            }
+
+            if (fileNameCompare < 0) {
+                uploadList.push(localFile);
+                sizeUpload += localFile.size ?? 0;
+                localPosition += 1;
+            }
+            else if (fileNameCompare > 0) {
+                deleteList.push(serverFile);
+                sizeDelete += serverFile.size ?? 0;
+                serverPosition += 1;
+            }
+            else if (fileNameCompare === 0) {
+                // paths are a match
+                if (localFile.type === "file" && serverFile.type === "file") {
+                    if (localFile.hash === serverFile.hash) {
+                        sameList.push(localFile);
+                    }
+                    else {
+                        sizeReplace += localFile.size ?? 0;
+                        replaceList.push(localFile);
+                    }
+                }
+
+                localPosition += 1;
+                serverPosition += 1;
+            }
+        }
+
+        // optimize modifications
+        let foldersToDelete = deleteList.filter((item) => item.type === "folder");
+
+        // remove files/folders that have a nested parent folder we plan on deleting
+        const optimizedDeleteList = deleteList.filter((itemToDelete) => {
+            const parentFolderIsBeingDeleted = foldersToDelete.find((folder) => {
+                const isSameFile = itemToDelete.name === folder.name;
+                const parentFolderExists = itemToDelete.name.startsWith(folder.name);
+
+                return parentFolderExists && !isSameFile;
+            }) !== undefined;
+
+            if (parentFolderIsBeingDeleted) {
+                // a parent folder is being deleted, no need to delete this one
+                return false;
+            }
+
+            return true;
+        });
+
+        return {
+            upload: uploadList,
+            delete: optimizedDeleteList,
+            replace: replaceList,
+            same: sameList,
+            sizeDelete,
+            sizeReplace,
+            sizeUpload
+        };
+    }
+}

package/vendor/ftp-deploy/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2020 SamKirkland
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/vendor/ftp-deploy/README.md

@@ -0,0 +1,3 @@
+Vendored from [benallfree/ftp-deploy](https://github.com/benallfree/ftp-deploy) @ `132389e` (fork of [SamKirkland/FTP-Deploy-Action](https://github.com/SamKirkland/FTP-Deploy-Action) sync engine).
+
+Incremental FTPS/SFTP deploy: hash diff, include/exclude globs, `.ftp-deploy-sync-state.json`. phio uses `protocol: sftp` via `ssh2-sftp-client`. See `LICENSE`.

package/vendor/ftp-deploy/deploy.ts

@@ -0,0 +1,226 @@
+import * as ftp from "basic-ftp";
+import fs from "fs";
+import { IFileList, IDiff, syncFileDescription, currentSyncFileVersion, IFtpDeployArgumentsWithDefaults } from "./types";
+import { HashDiff } from "./HashDiff";
+import { ILogger, retryRequest, ITimings, applyExcludeFilter, formatNumber } from "./utilities";
+import prettyBytes from "pretty-bytes";
+import { prettyError } from "./errorHandling";
+import { ensureDir, FTPSyncProvider } from "./syncProvider";
+import { getLocalFiles } from "./localFiles";
+import { deploySftp } from "./sftpDeploy";
+
+async function downloadFileList(client: ftp.Client, logger: ILogger, path: string): Promise<IFileList> {
+    // note: originally this was using a writable stream instead of a buffer file
+    // basic-ftp doesn't seam to close the connection when using steams over some ftps connections. This appears to be dependent on the ftp server
+    const tempFileNameHack = ".ftp-deploy-sync-server-state-buffer-file---delete.json";
+
+    await retryRequest(logger, async () => await client.downloadTo(tempFileNameHack, path));
+
+    const fileAsString = fs.readFileSync(tempFileNameHack, { encoding: "utf-8" });
+    const fileAsObject = JSON.parse(fileAsString) as IFileList;
+
+    fs.unlinkSync(tempFileNameHack);
+
+    return fileAsObject;
+}
+
+function createLocalState(localFiles: IFileList, logger: ILogger, args: IFtpDeployArgumentsWithDefaults): void {
+    logger.verbose(`Creating local state at ${args["local-dir"]}${args["state-name"]}`);
+    fs.writeFileSync(`${args["local-dir"]}${args["state-name"]}`, JSON.stringify(localFiles, undefined, 4), { encoding: "utf8" });
+    logger.verbose("Local state created");
+}
+
+async function connect(client: ftp.Client, args: IFtpDeployArgumentsWithDefaults, logger: ILogger) {
+    let secure: boolean | "implicit" = false;
+    if (args.protocol === "ftps") {
+        secure = true;
+    }
+    else if (args.protocol === "ftps-legacy") {
+        secure = "implicit";
+    }
+
+    client.ftp.verbose = args["log-level"] === "verbose";
+
+    const rejectUnauthorized = args.security === "strict";
+
+    try {
+        await client.access({
+            host: args.server,
+            user: args.username,
+            password: args.password,
+            port: args.port,
+            secure: secure,
+            secureOptions: {
+                rejectUnauthorized: rejectUnauthorized
+            }
+        });
+    }
+    catch (error) {
+        logger.all("Failed to connect, are you sure your server works via FTP or FTPS? Users sometimes get this error when the server only supports SFTP.");
+        throw error;
+    }
+
+    if (args["log-level"] === "verbose") {
+        client.trackProgress(info => {
+            logger.verbose(`${info.type} progress for "${info.name}". Progress: ${info.bytes} bytes of ${info.bytesOverall} bytes`);
+        });
+    }
+}
+
+export async function getServerFiles(client: ftp.Client, logger: ILogger, timings: ITimings, args: IFtpDeployArgumentsWithDefaults): Promise<IFileList> {
+    try {
+        await ensureDir(client, logger, timings, args["server-dir"]);
+
+        if (args["dangerous-clean-slate"]) {
+            logger.all(`----------------------------------------------------------------`);
+            logger.all("🗑️ Removing all files on the server because 'dangerous-clean-slate' was set, this will make the deployment very slow...");
+            if (args["dry-run"] === false) {
+                await client.clearWorkingDir();
+            }
+            logger.all("Clear complete");
+
+            throw new Error("dangerous-clean-slate was run");
+        }
+
+        const serverFiles = await downloadFileList(client, logger, args["state-name"]);
+        logger.all(`----------------------------------------------------------------`);
+        logger.all(`Last published on 📅 ${new Date(serverFiles.generatedTime).toLocaleDateString(undefined, { weekday: "long", year: "numeric", month: "long", day: "numeric", hour: "numeric", minute: "numeric" })}`);
+
+        // apply exclude options to server
+        if (args.exclude.length > 0) {
+            const filteredData = serverFiles.data.filter((item) => applyExcludeFilter({ path: item.name, isDirectory: () => item.type === "folder" }, args.exclude));
+            serverFiles.data = filteredData;
+        }
+
+        return serverFiles;
+    }
+    catch (error) {
+        logger.all(`----------------------------------------------------------------`);
+        logger.all(`No file exists on the server "${args["server-dir"] + args["state-name"]}" - this must be your first publish! 🎉`);
+        logger.all(`The first publish will take a while... but once the initial sync is done only differences are published!`);
+        logger.all(`If you get this message and its NOT your first publish, something is wrong.`);
+
+        // set the server state to nothing, because we don't know what the server state is
+        return {
+            description: syncFileDescription,
+            version: currentSyncFileVersion,
+            generatedTime: new Date().getTime(),
+            data: [],
+        };
+    }
+}
+
+export async function deploy(args: IFtpDeployArgumentsWithDefaults, logger: ILogger, timings: ITimings): Promise<void> {
+    if (args.protocol === "sftp") {
+        return deploySftp(args, logger, timings);
+    }
+
+    timings.start("total");
+
+    // header
+    logger.all(`----------------------------------------------------------------`);
+    logger.all(`🚀 Thanks for using ftp-deploy. Let's deploy some stuff!   `);
+    logger.all(`----------------------------------------------------------------`);
+    logger.all(`If you found this project helpful, please support it`);
+    logger.all(`by giving it a ⭐ on Github --> https://github.com/SamKirkland/FTP-Deploy-Action`);
+    logger.all(`or add a badge 🏷️ to your projects readme --> https://github.com/SamKirkland/FTP-Deploy-Action#badge`);
+    logger.verbose(`Using the following include filters: ${JSON.stringify(args.include)}`);
+    logger.verbose(`Using the following excludes filters: ${JSON.stringify(args.exclude)}`);
+
+    timings.start("hash");
+    const localFiles = await getLocalFiles(args, logger);
+    timings.stop("hash");
+
+    createLocalState(localFiles, logger, args);
+
+    const client = new ftp.Client(args.timeout);
+
+    global.reconnect = async function () {
+        timings.start("connecting");
+        await connect(client, args, logger);
+        timings.stop("connecting");
+    }
+
+
+    let totalBytesUploaded = 0;
+    try {
+        await global.reconnect();
+
+        const serverFiles = await getServerFiles(client, logger, timings, args);
+
+        timings.start("logging");
+        const diffTool: IDiff = new HashDiff();
+
+        logger.standard(`----------------------------------------------------------------`);
+        logger.standard(`Local Files:\t${formatNumber(localFiles.data.length)}`);
+        logger.standard(`Server Files:\t${formatNumber(serverFiles.data.length)}`);
+        logger.standard(`----------------------------------------------------------------`);
+        logger.standard(`Calculating differences between client & server`);
+        logger.standard(`----------------------------------------------------------------`);
+        logger.verbose(`Local files:`, JSON.stringify(localFiles,null,2));
+        logger.verbose(`Server files:`, JSON.stringify(serverFiles,null,2));
+
+        const diffs = diffTool.getDiffs(localFiles, serverFiles);
+
+        diffs.upload.filter((itemUpload) => itemUpload.type === "folder").map((itemUpload) => {
+            logger.standard(`📁 Create: ${itemUpload.name}`);
+        });
+
+        diffs.upload.filter((itemUpload) => itemUpload.type === "file").map((itemUpload) => {
+            logger.standard(`📄 Upload: ${itemUpload.name}`);
+        });
+
+        diffs.replace.map((itemReplace) => {
+            logger.standard(`🔁 File replace: ${itemReplace.name}`);
+        });
+
+        diffs.delete.filter((itemUpload) => itemUpload.type === "file").map((itemDelete) => {
+            logger.standard(`📄 Delete: ${itemDelete.name}    `);
+        });
+
+        diffs.delete.filter((itemUpload) => itemUpload.type === "folder").map((itemDelete) => {
+            logger.standard(`📁 Delete: ${itemDelete.name}    `);
+        });
+
+        diffs.same.map((itemSame) => {
+            if (itemSame.type === "file") {
+                logger.standard(`⚖️  File content is the same, doing nothing: ${itemSame.name}`);
+            }
+        });
+        timings.stop("logging");
+
+
+        totalBytesUploaded = diffs.sizeUpload + diffs.sizeReplace;
+
+        timings.start("upload");
+        try {
+            const syncProvider = new FTPSyncProvider(client, logger, timings, args["local-dir"], args["server-dir"], args["state-name"], args["dry-run"]);
+            await syncProvider.syncLocalToServer(diffs);
+        }
+        finally {
+            timings.stop("upload");
+        }
+    }
+    catch (error) {
+        prettyError(logger, args, error);
+        throw error;
+    }
+    finally {
+        client.close();
+        timings.stop("total");
+    }
+
+
+    const uploadSpeed = prettyBytes(totalBytesUploaded / (timings.getTime("upload") / 1000));
+
+    // footer
+    logger.all(`----------------------------------------------------------------`);
+    logger.all(`Time spent hashing: ${timings.getTimeFormatted("hash")}`);
+    logger.all(`Time spent connecting to server: ${timings.getTimeFormatted("connecting")}`);
+    logger.all(`Time spent deploying: ${timings.getTimeFormatted("upload")} (${uploadSpeed}/second)`);
+    logger.all(`  - changing dirs: ${timings.getTimeFormatted("changingDir")}`);
+    logger.all(`  - logging: ${timings.getTimeFormatted("logging")}`);
+    logger.all(`----------------------------------------------------------------`);
+    logger.all(`Total time: ${timings.getTimeFormatted("total")}`);
+    logger.all(`----------------------------------------------------------------`);
+}

package/vendor/ftp-deploy/errorHandling.ts

@@ -0,0 +1,67 @@
+import { ILogger } from "./utilities";
+import { IFtpDeployArgumentsWithDefaults, ErrorCode } from "./types";
+import { FTPError } from "basic-ftp";
+
+
+function logOriginalError(logger: ILogger, error: any) {
+    logger.all();
+    logger.all(`----------------------------------------------------------------`);
+    logger.all(`----------------------  full error below  ----------------------`);
+    logger.all(`----------------------------------------------------------------`);
+    logger.all();
+    logger.all(error);
+}
+
+
+/**
+ * Converts a exception to helpful debug info
+ * @param error exception
+ */
+export function prettyError(logger: ILogger, args: IFtpDeployArgumentsWithDefaults, error: any): void {
+    logger.all();
+    logger.all(`----------------------------------------------------------------`);
+    logger.all(`--------------  🔥🔥🔥 an error occurred  🔥🔥🔥  --------------`);
+    logger.all(`----------------------------------------------------------------`);
+
+    const ftpError = error as FTPError;
+    if (typeof error.code === "string") {
+        const errorCode = error.code as string;
+
+        if (errorCode === "ENOTFOUND") {
+            logger.all(`The server "${args.server}" doesn't seem to exist. Do you have a typo?`);
+        }
+    }
+    else if (typeof error.name === "string") {
+        const errorName = error.name as string;
+
+        if (errorName.includes("ERR_TLS_CERT_ALTNAME_INVALID")) {
+            logger.all(`The certificate for "${args.server}" is likely shared. The host did not place your server on the list of valid domains for this cert.`);
+            logger.all(`This is a common issue with shared hosts. You have a few options:`);
+            logger.all(` - Ignore this error by setting security back to loose`);
+            logger.all(` - Contact your hosting provider and ask them for your servers hostname`);
+        }
+    }
+    else if (args.protocol === "sftp") {
+        const message = `${error?.message ?? error}`.toLowerCase();
+        if (message.includes("authentication") || message.includes("all configured")) {
+            logger.all(`Could not authenticate over SFTP with username "${args.username}".`);
+            logger.all(`Ensure the Phio deploy key is registered under Account → Keys.`);
+        }
+    }
+    else if (typeof ftpError.code === "number") {
+        if (ftpError.code === ErrorCode.NotLoggedIn) {
+            const serverRequiresFTPS = ftpError.message.toLowerCase().includes("must use encryption");
+
+            if (serverRequiresFTPS) {
+                logger.all(`The server you are connecting to requires encryption (ftps)`);
+                logger.all(`Enable FTPS by using the protocol option.`);
+            }
+            else {
+                logger.all(`Could not login with the username "${args.username}" and password "${args.password}".`);
+                logger.all(`Make sure you can login with those credentials. If you have a space or a quote in your username or password be sure to escape them!`);
+            }
+        }
+    }
+
+    logOriginalError(logger, error);
+}

package/vendor/ftp-deploy/localFiles.ts

@@ -0,0 +1,47 @@
+import { Record, IFileList, syncFileDescription, currentSyncFileVersion, IFtpDeployArgumentsWithDefaults } from "./types";
+import { fileHash } from "./HashDiff";
+import { globSync} from 'glob'
+import { lstatSync } from "fs";
+import { ILogger } from "./utilities";
+
+export async function getLocalFiles(args: IFtpDeployArgumentsWithDefaults, logger: ILogger): Promise<IFileList> {
+    const files = globSync(args['include'], {ignore: args['exclude'], cwd: args['local-dir'], });
+    logger.verbose(`Local files:`, JSON.stringify({files},null,2));
+
+    const records: Record[] = [];
+
+    for (const path of files) {
+        const stat = lstatSync(path);
+        if (stat.isDirectory()) {
+            records.push({
+                type: "folder",
+                name: path,
+                size: undefined
+            });
+
+            continue;
+        }
+
+        if (stat.isFile()) {
+            records.push({
+                type: "file",
+                name: path,
+                size: stat.size,
+                hash: await fileHash(args["local-dir"] + path, "sha256")
+            });
+
+            continue;
+        }
+
+        if (stat.isSymbolicLink()) {
+            console.warn("This script is currently unable to handle symbolic links - please add a feature request if you need this");
+        }
+    }
+
+    return {
+        description: syncFileDescription,
+        version: currentSyncFileVersion,
+        generatedTime: new Date().getTime(),
+        data: records
+    };
+}

package/vendor/ftp-deploy/module.ts

@@ -0,0 +1,28 @@
+import { deploy as deployCustom } from "./deploy";
+import { IFtpDeployArguments } from "./types";
+import { getDefaultSettings, Logger, Timings } from "./utilities";
+
+/**
+ * Default includes, everything by default
+ */
+export const includeDefaults = ["**/*"];
+
+/**
+ * Default excludes, ignores all git files and the node_modules folder
+ * **\/.git* ignores all FILES that start with .git(in any folder or sub-folder)
+ * **\/.git*\/** ignores all FOLDERS that start with .git (in any folder or sub-folder)
+ * **\/node_modules\/** ignores all FOLDERS named node_modules (in any folder or sub-folder)
+ */
+export const excludeDefaults = ["**/.git*", "**/.git*/**", "**/node_modules/**"];
+
+/**
+ * Syncs a local folder with a remote folder over FTP. 
+ * After the initial sync only differences are synced, making deployments super fast!
+ */
+export async function deploy(args: IFtpDeployArguments): Promise<void> {
+  const argsWithDefaults = getDefaultSettings(args);
+  const logger = new Logger(argsWithDefaults["log-level"]);
+  const timings = new Timings();
+
+  await deployCustom(argsWithDefaults, logger, timings);
+}