spec-3nweb-server 1.8.5 → 1.8.7

package/ts-code/tests/libs-for-tests/server-components/3nstorage.ts

@@ -0,0 +1,206 @@
+/*
+ Copyright (C) 2016, 2019 - 2020 3NSoft Inc.
+ 
+ This program is free software: you can redistribute it and/or modify it under
+ the terms of the GNU General Public License as published by the Free Software
+ Foundation, either version 3 of the License, or (at your option) any later
+ version.
+ 
+ This program is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ See the GNU General Public License for more details.
+ 
+ You should have received a copy of the GNU General Public License along with
+ this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+import { Component } from './component';
+import { addUserIn } from './mailerid';
+import { User } from '../pkl';
+import { resolve as resolveUrl } from 'url';
+import { get3NStorageServiceUrl } from '../3nstorage';
+import * as fs from '../../../lib-common/async-fs-node';
+import { Obj } from '../3nstorage';
+import { bytesEqual } from '../bytes-equal';
+import { ObjStatusInfo } from '../../../3nstorage/resources/store';
+import { ObjVersionFile } from '../../../lib-common/objs-on-disk/obj-file';
+import { join } from 'path';
+import { joinByteArrs } from '../../../lib-common/buffer-utils';
+import { userDataInRootFolder } from '../../../lib-server/resources/server-data-folders';
+
+async function rmFolderContent(folder: string): Promise<void> {
+	await fs.rmDirWithContent(folder);
+	await fs.mkdir(folder);
+}
+
+export class StorageComponent extends Component {
+	
+	constructor(
+		signupDomains: string[],
+		public midServiceDomain: string
+	) {
+		super(
+			{
+				rootFolder: (null as any),
+				domain: midServiceDomain,
+				signup: {},
+				enabledServices: {
+					mailerId: true,
+					storage: true
+				},
+				mailerId: {
+					certs: (null as any)
+				}
+			},
+			signupDomains
+		);
+		Object.seal(this);
+	}
+	
+	async addUser(userId: string, configureDNS = true): Promise<User> {
+		if (configureDNS) { this.dns.configureFor(userId); }
+		let user = await addUserIn(this.url, userId);
+		user.midUrl = this.midUrl;
+		user.storageOwnerUrl = await this.getStorageOwnerUrl();
+		return user;
+	}
+	
+	get midUrl(): string {
+		return resolveUrl(this.url, 'mailerid/');
+	}
+	
+	get storageUrl(): string {
+		return resolveUrl(this.url, '3nstorage/');
+	}
+	
+	async getStorageOwnerUrl(): Promise<string> {
+		let storageOwnerUrl = await get3NStorageServiceUrl(
+			this.storageUrl, 'owner');
+		if (!storageOwnerUrl.endsWith('/')) { storageOwnerUrl += '/'; }
+		return storageOwnerUrl;
+	}
+	
+	private storeFolder(userId: string): string {
+		return join(userDataInRootFolder(this.conf.rootFolder, userId), 'store');
+	}
+	
+	private async checkTransactionPresence(
+		transFolder: string, transactionId: string|undefined
+	): Promise<boolean> {
+		const transFile = 'transaction';
+		try {
+			const lst = await fs.readdir(transFolder);
+			expect(lst).toContain(transFile);
+		} catch (exc) {
+			if ((exc as fs.FileException).notFound) { return false; }
+			else { throw exc; }
+		}
+		if (transactionId) {
+			const trans = JSON.parse(await fs.readFile(
+				join(transFolder, transFile), { encoding: 'utf8' }));
+			return (trans.transactionId === transactionId);
+		}
+		return true;
+	}
+	
+	rootTransactionExists(
+		userId: string, transactionId?: string
+	): Promise<boolean> {
+		return this.checkTransactionPresence(
+			join(this.storeFolder(userId), 'root', 'transaction'), transactionId);
+	}
+	
+	transactionExists(
+		userId: string, objId: string, transactionId?: string
+	): Promise<boolean> {
+		return this.checkTransactionPresence(
+			join(this.storeFolder(userId),'transactions', objId), transactionId);
+	}
+	
+	private async checkCurrentObjPresence(
+		objFolder: string, ver: number|undefined, obj: Obj|undefined
+	): Promise<boolean> {
+		let status: ObjStatusInfo;
+		try {
+			status = JSON.parse(await fs.readFile(
+				objFolder+'/status', { encoding: 'utf8' }));
+		} catch (exc) {
+			if ((<fs.FileException> exc).notFound) { return false; }
+			else { throw exc; }
+		}
+		if (status.state !== 'current') { return false; }
+		const currVer = status.currentVersion;
+		if ((ver !== undefined) && (currVer !== ver)) { return false; }
+		if (obj) {
+			const filePath = join(objFolder, `${currVer}.v`);
+			const file = await ObjVersionFile.forExisting(filePath)
+			.catch((exc: fs.FileException) => {
+				if (exc.notFound) { return undefined; }
+				throw exc;
+			});
+			if (!file) { return false; }
+			const headerInFile = await file.readHeader();
+			if (!headerInFile
+			|| !bytesEqual(headerInFile, obj.header)) { return false; }
+			const diff = obj.diff;
+			if (file.getBaseVersion() === undefined) {
+				if (diff) { return false; }
+				const segsInFile = joinByteArrs(
+					await file.readSegs(0, file.getTotalSegsLen()));
+				if (!bytesEqual(segsInFile, obj.segs)) { return false; }
+			} else {
+				if (!diff) { return false; }
+				if (file.getBaseVersion() !== diff.baseVersion) { return false; }
+				if (file.getTotalSegsLen() !== diff.segsSize) { return false; }
+				let ofs = 0;
+				for (const [ isNew, dsOfs, len ] of diff.sections) {
+					if (isNew === 0) {
+						const sections = await file.segsLocations(ofs, len);
+						for (const section of sections) {
+							if (section.type !== 'base') { return false; }
+						}
+					} else {
+						if (!bytesEqual(
+							joinByteArrs(await file.readSegs(ofs, len)),
+							obj.segs.subarray(dsOfs, dsOfs+len))) { return false; }
+					}
+					ofs += len;
+				}
+			}
+		}
+		return true;
+	}
+	
+	currentRootObjExists(
+		userId: string, ver?: number, obj?: Obj
+	): Promise<boolean> {
+		return this.checkCurrentObjPresence(
+			join(this.storeFolder(userId), 'root'), ver, obj);
+	}
+	
+	currentObjExists(
+		userId: string, objId: string, ver?: number, obj?: Obj
+	): Promise<boolean> {
+		return this.checkCurrentObjPresence(
+			join(this.storeFolder(userId), 'objects', objId), ver, obj);
+	}
+
+	async clearStorage(userId: string): Promise<void> {
+		let storeFolder = this.storeFolder(userId);
+		await rmFolderContent(join(storeFolder, 'transactions'));
+		await rmFolderContent(join(storeFolder, 'objects'));
+		await rmFolderContent(join(storeFolder, 'root'));
+	}
+
+	async restartAndClearStorageFor(userId: string): Promise<void> {
+		await this.clearStorage(userId);
+		await this.restart();
+	}
+	
+}
+Object.freeze(StorageComponent.prototype);
+Object.freeze(StorageComponent);
+
+
+Object.freeze(exports);

package/ts-code/tests/libs-for-tests/server-components/admin.ts

@@ -0,0 +1,49 @@
+/*
+ Copyright (C) 2016, 2020 3NSoft Inc.
+ 
+ This program is free software: you can redistribute it and/or modify it under
+ the terms of the GNU General Public License as published by the Free Software
+ Foundation, either version 3 of the License, or (at your option) any later
+ version.
+ 
+ This program is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ See the GNU General Public License for more details.
+ 
+ You should have received a copy of the GNU General Public License along with
+ this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+import { Component } from './component';
+import { resolve as resolveUrl } from 'url';
+import { addSingleUserSignup } from '../../../lib';
+
+export class AdminComponent extends Component {
+	
+	constructor(signupDomains: string[]) {
+		super(
+			{
+				rootFolder: (null as any),
+				domain: (null as any),
+				signup: {},
+				enabledServices: {}
+			},
+			signupDomains
+		);
+		Object.seal(this);
+	}
+	
+	get signupUrl(): string {
+		return resolveUrl(this.url, 'signup/');
+	}
+
+	createSingleUserSignupTokenFor(userId: string): Promise<string> {
+		return addSingleUserSignup(this.dataFolder, userId);
+	}
+	
+}
+Object.freeze(AdminComponent.prototype);
+Object.freeze(AdminComponent);
+
+Object.freeze(exports);

package/ts-code/tests/libs-for-tests/server-components/asmail.ts

@@ -0,0 +1,152 @@
+/*
+ Copyright (C) 2016 - 2017, 2019 - 2020, 2025 3NSoft Inc.
+ 
+ This program is free software: you can redistribute it and/or modify it under
+ the terms of the GNU General Public License as published by the Free Software
+ Foundation, either version 3 of the License, or (at your option) any later
+ version.
+ 
+ This program is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ See the GNU General Public License for more details.
+ 
+ You should have received a copy of the GNU General Public License along with
+ this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+import { Component } from './component';
+import { addUserIn } from './mailerid';
+import { User } from '../pkl';
+import { resolve as resolveUrl } from 'url';
+import { getASMailServiceUrl } from '../asmail';
+import * as fs from '../../../lib-common/async-fs-node';
+import { Msg } from '../asmail';
+import * as retrievalApi from '../../../lib-common/service-api/asmail/retrieval';
+import { bytesEqual } from '../bytes-equal';
+import { join } from 'path';
+import { ObjVersionFile } from '../../../lib-common/objs-on-disk/obj-file';
+import { joinByteArrs } from '../../../lib-common/buffer-utils';
+import { userDataInRootFolder } from '../../../lib-server/resources/server-data-folders';
+
+export class ASMailComponent extends Component {
+	
+	constructor(
+		signupDomains: string[],
+		public midServiceDomain: string
+	) {
+		super(
+			{
+				rootFolder: (null as any),
+				domain: midServiceDomain,
+				signup: {},
+				enabledServices: {
+					mailerId: true,
+					asmail: true
+				},
+				mailerId: {
+					certs: (null as any)
+				}
+			},
+			signupDomains
+		);
+		Object.seal(this);
+	}
+	
+	async addUser(userId: string, configureDNS = true): Promise<User> {
+		if (configureDNS) { this.dns.configureFor(userId); }
+		let user = await addUserIn(this.url, userId);
+		user.midUrl = this.midUrl;
+		return user;
+	}
+	
+	get midUrl(): string {
+		return resolveUrl(this.url, 'mailerid/');
+	}
+	
+	get asmailUrl(): string {
+		return resolveUrl(this.url, 'asmail/');
+	}
+	
+	getConfUrl(): Promise<string> {
+		return getASMailServiceUrl(this.asmailUrl, 'config');
+	}
+	
+	getDeliveryUrl(): Promise<string> {
+		return getASMailServiceUrl(this.asmailUrl, 'delivery');
+	}
+	
+	getRetrievalUrl(): Promise<string> {
+		return getASMailServiceUrl(this.asmailUrl, 'retrieval');
+	}
+	
+	private inboxFolder(userId: string): string {
+		return join(userDataInRootFolder(this.conf.rootFolder, userId), 'mail');
+	}
+
+	private msgFolder(
+		userId: string, msgId: string, deliveryComplete: boolean
+	): string {
+		return join(this.inboxFolder(userId),
+			(deliveryComplete ? 'messages' : 'delivery'), msgId);
+	}
+	
+	// XXX adopt new file access
+	async msgExists(userId: string, msgId: string, deliveryComplete: boolean,
+			msg?: Msg): Promise<boolean> {
+		let meta: retrievalApi.msgMetadata.Reply
+		let msgFolder = this.msgFolder(userId, msgId, deliveryComplete);
+		try {
+			let metaFile = join(msgFolder, 'meta.json');
+			meta = JSON.parse(await fs.readFile(metaFile, { encoding: 'utf8' }));
+		} catch (exc) {
+			if ((exc as fs.FileException).notFound) { return false; }
+			else { throw exc; }
+		}
+		let extMeta = meta.extMeta;
+		if (!deliveryComplete || ! msg) { return true; }
+
+		// check meta
+		if (msg.cryptoMeta.pid) {
+			expect(extMeta.pid).withContext('correct key-pair id in meta').toBe(msg.cryptoMeta.pid);
+		}
+		if (msg.cryptoMeta.recipientKid) {
+			expect(extMeta.recipientKid).withContext('correct recipient key id in meta').toBe(msg.cryptoMeta.recipientKid);
+		}
+		if (msg.cryptoMeta.senderPKey) {
+			expect(extMeta.senderPKey).withContext('correct sender\'s public key').toBe(msg.cryptoMeta.senderPKey);
+		}
+		expect(extMeta.objIds.length).withContext('correct number of message objects in meta').toBe(msg.msgObjs.length);
+		expect(Object.keys(meta.objs).length).withContext('correct number of message object sizes in meta').toBe(msg.msgObjs.length);
+		
+		for (let i=0; i < msg.msgObjs.length; i+=1) {
+			// check meta info for each object
+			let obj = msg.msgObjs[i];
+			let objId = obj.objId!;
+			expect(extMeta.objIds[i]).toBe(objId);
+			expect(meta.objs[objId].size.header).withContext('header length of a message object').toBe(obj.header.length);
+			expect(meta.objs[objId].size.segments).withContext('segments length of a message object').toBe(obj.segs.length);
+			expect(meta.objs[objId].completed).withContext('completness flag of a message object').toBe(true);
+
+			// check object file
+			const filePath = join(msgFolder, objId);
+			const file = await ObjVersionFile.forExisting(filePath)
+			.catch((exc: fs.FileException) => {
+				if (exc.notFound) { return undefined; }
+				throw exc;
+			});
+			if (!file) { return false; }
+			const headerInFile = await file.readHeader();
+			if (!headerInFile
+			|| !bytesEqual(headerInFile, obj.header)) { return false; }
+			const segsInFile = joinByteArrs(
+				await file.readSegs(0, file.getTotalSegsLen()));
+			if (!bytesEqual(segsInFile, obj.segs)) { return false; }
+		}
+
+		return true;
+	}
+	
+}
+Object.freeze(ASMailComponent.prototype);
+Object.freeze(ASMailComponent);

package/ts-code/tests/libs-for-tests/server-components/component.ts

@@ -0,0 +1,90 @@
+/*
+ Copyright (C) 2016, 2020 3NSoft Inc.
+ 
+ This program is free software: you can redistribute it and/or modify it under
+ the terms of the GNU General Public License as published by the Free Software
+ Foundation, either version 3 of the License, or (at your option) any later
+ version.
+ 
+ This program is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ See the GNU General Public License for more details.
+ 
+ You should have received a copy of the GNU General Public License along with
+ this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+import { ServerRunner } from '../server-runner';
+import { mkdir, rmDirWithContent, writeFile } from '../../../lib-common/async-fs-node';
+import { FileException } from '../../../lib-common/exceptions/file';
+import { accountsApp as makeAdminApp, Configurations, servicesApp as makeServicesApp } from '../../../services';
+import { DNSMock } from '../../../mock/dns';
+import * as dns from 'dns';
+import { AppWithWSs } from '../../../lib-server/web-sockets/app';
+import { join } from 'path';
+import { makeMultiDomainSignupCtx } from '../../../signup/signup-tokens';
+
+const DEFAULT_SERVICE_PORT = 8088;
+const DEFAULT_DATA_FOLDER = join(__dirname, '/../../../../test-data');
+
+export abstract class Component extends ServerRunner {
+
+	dns: DNSMock;
+
+	constructor(
+		protected conf: Configurations,
+		private signupDomains: string[],
+		servicePort = DEFAULT_SERVICE_PORT,
+		protected dataFolder = DEFAULT_DATA_FOLDER
+	) {
+		super(servicePort);
+		this.conf.rootFolder = this.dataFolder;
+		this.conf.signup!.noTokenFile = join(
+			this.dataFolder, 'no-tokens-signup.json');
+		if (this.conf.mailerId) {
+			const certs = this.conf.mailerId.certs;
+			this.conf.mailerId.certs = join(
+				this.dataFolder, (certs ? certs : 'mid-certs.json'));
+		}
+		const urlWithoutProto = this.url.substring(this.url.indexOf('://')+3);
+		this.dns = new DNSMock(urlWithoutProto+'mailerid/');
+	}
+
+	async prepToRun(): Promise<AppWithWSs> {
+
+		// prepare data folders
+		try {
+			await mkdir(this.dataFolder);
+			// write to-tokens signup file
+			const noTokenCtx = makeMultiDomainSignupCtx(this.signupDomains);
+			await writeFile(
+				this.conf.signup!.noTokenFile!, JSON.stringify(noTokenCtx),
+				{ encoding: 'utf8' }
+			);
+		} catch (exc) {
+			if (!(exc as FileException).alreadyExists) { throw exc; }
+		}
+
+		// inject dns mock
+		(dns as any).resolveTxt = this.dns.resolveTxt;
+
+		// setup servers
+		const app = new AppWithWSs();
+		app.use(makeAdminApp(this.conf));
+		app.use(makeServicesApp(this.conf));
+
+		return app;
+	}
+
+	async clean(): Promise<void> {
+		await rmDirWithContent(this.dataFolder).catch((exc: FileException) => {
+			if (!exc.notFound) { throw exc; }
+		});
+	}
+
+}
+Object.freeze(Component.prototype);
+Object.freeze(Component);
+
+Object.freeze(exports);

package/ts-code/tests/libs-for-tests/server-components/mailerid.ts

@@ -0,0 +1,114 @@
+/*
+ Copyright (C) 2016, 2020 3NSoft Inc.
+ 
+ This program is free software: you can redistribute it and/or modify it under
+ the terms of the GNU General Public License as published by the Free Software
+ Foundation, either version 3 of the License, or (at your option) any later
+ version.
+ 
+ This program is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ See the GNU General Public License for more details.
+ 
+ You should have received a copy of the GNU General Public License along with
+ this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+import { Component } from './component';
+import { resolve as resolveUrl } from 'url';
+import { doJsonRequest, RequestOpts } from '../xhr-utils';
+import * as signup from '../../../lib-common/admin-api/signup';
+import { User } from '../pkl';
+import * as random from '../../../lib-common/random-node';
+import { box } from 'ecma-nacl';
+import { use } from '../../../lib-common/jwkeys';
+import { base64 } from '../../../lib-common/buffer-utils';
+import { getProvPath } from '../mailerid';
+
+export async function addUserIn(rootServerUrl: string, userId: string): Promise<User> {
+	let reqOpts: RequestOpts = {
+		url: resolveUrl(resolveUrl(
+			rootServerUrl, 'signup/'), signup.addUser.URL_END),
+		method: 'POST'
+	};
+	let user: User = {
+		id: userId,
+		loginDefaultSKey: await random.bytes(box.KEY_LENGTH),
+		loginLabeledSKey: {
+			k: await random.bytes(box.KEY_LENGTH),
+			kid: await random.stringOfB64Chars(12)
+		},
+		midUrl: (undefined as any),
+		storageOwnerUrl: (undefined as any)
+	};
+	let req: signup.addUser.Request = {
+		userId: user.id,
+		storage: {
+			kdParams: {}
+		},
+		mailerId: {
+			defaultPKey: {
+				pkey: {
+					alg: box.JWK_ALG_NAME,
+					kid: '_',
+					use: use.MID_PKLOGIN,
+					k: base64.pack(box.generate_pubkey(user.loginDefaultSKey))
+				},
+				kdParams: {}
+			},
+			otherPKeys: [ {
+					alg: box.JWK_ALG_NAME,
+					kid: user.loginLabeledSKey.kid,
+					use: use.MID_PKLOGIN,
+					k: base64.pack(box.generate_pubkey(user.loginLabeledSKey.k))
+				} ]
+		}
+	};
+	let rep = await doJsonRequest<string[]>(reqOpts, req);
+	if (rep.status !== signup.addUser.SC.ok) {
+		throw new Error(`Fail to add user ${userId}`);
+	}
+	return user;
+}
+
+export class MailerIdComponent extends Component {
+	
+	constructor(
+		signupDomains: string[],
+		public midServiceDomain: string
+	) {
+		super(
+			{
+				rootFolder: (null as any),
+				domain: midServiceDomain,
+				signup: {},
+				enabledServices: {
+					mailerId: true
+				},
+				mailerId: {
+					certs: (null as any)
+				}
+			},
+			signupDomains
+		);
+		Object.seal(this);
+	}
+	
+	addUser(userId: string): Promise<User> {
+		return addUserIn(this.url, userId);
+	}
+	
+	get midUrl(): string {
+		return resolveUrl(this.url, 'mailerid/');
+	}
+	
+	getProvUrl(): Promise<string> {
+		return getProvPath(this.midUrl);
+	}
+	
+}
+Object.freeze(MailerIdComponent.prototype);
+Object.freeze(MailerIdComponent);
+
+Object.freeze(exports);

package/ts-code/tests/libs-for-tests/server-runner.ts

@@ -0,0 +1,82 @@
+/*
+ Copyright (C) 2016 3NSoft Inc.
+ 
+ This program is free software: you can redistribute it and/or modify it under
+ the terms of the GNU General Public License as published by the Free Software
+ Foundation, either version 3 of the License, or (at your option) any later
+ version.
+ 
+ This program is distributed in the hope that it will be useful, but
+ WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ See the GNU General Public License for more details.
+ 
+ You should have received a copy of the GNU General Public License along with
+ this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+import * as https from "https";
+import { sslOpts } from '../../mock/tls-options';
+import { AppWithWSs } from '../../lib-server/web-sockets/app';
+import { sleep } from "../../lib-common/processes";
+
+// allow client test calls to trust above self-signed cert
+https.globalAgent.options.ca = sslOpts.cert;
+
+export type ServerState = 'stopped' | 'starting' | 'running' | 'stopping';
+
+export abstract class ServerRunner {
+	
+	protected state: ServerState = 'stopped';
+	protected app: AppWithWSs = (undefined as any);
+	
+	constructor(
+		protected servicePort: number
+	) {}
+	
+	abstract prepToRun(): Promise<AppWithWSs>;
+	
+	abstract clean(): Promise<void>;
+	
+	isRunning(): boolean {
+		return (this.state === 'running');
+	}
+
+	async start(): Promise<void> {
+		if (this.state !== 'stopped') { throw new Error(
+			`Home server is already in ${this.state} state.`); }
+		this.state = 'starting';
+		this.app = await this.prepToRun();
+		await this.app.start(sslOpts, this.servicePort);
+		await sleep(100);
+		this.state = 'running';
+	}
+	
+	async stop(doCleanup = true): Promise<void> {
+		if (this.state !== 'running') { throw new Error(
+			`Home server is in ${this.state} state instead of running state`); }
+		this.state = 'stopping';
+		await this.app.stop();
+		this.app = (undefined as any);
+		if (doCleanup) {
+			await this.clean();
+		}
+		this.state = 'stopped';
+	}
+
+	async restart(): Promise<void> {
+		if (this.isRunning()) {
+			await this.stop(false);
+		}
+		await this.start();
+	}
+	
+	protected get url(): string {
+		return `https://localhost:${this.servicePort}/`;
+	}
+	
+}
+Object.freeze(ServerRunner.prototype);
+Object.freeze(ServerRunner);
+
+Object.freeze(exports);