web_plsql 0.5.0 → 0.6.0

package/src/procedureSanitize.js

@@ -0,0 +1,215 @@
+import debugModule from 'debug';
+const debug = debugModule('webplsql:procedureSanitize');
+
+import oracledb from 'oracledb';
+import z from 'zod';
+import {RequestError} from './requestError.js';
+import {errorToString} from './error.js';
+
+/**
+ * @typedef {import('express').Request} Request
+ * @typedef {import('express').Response} Response
+ * @typedef {import('oracledb').Connection} Connection
+ * @typedef {import('oracledb').Result<unknown>} Result
+ * @typedef {import('./types.js').argObjType} argObjType
+ * @typedef {import('./types.js').fileUploadType} fileUploadType
+ * @typedef {import('./types.js').environmentType} environmentType
+ * @typedef {import('./types.js').configPlSqlHandlerType} configPlSqlHandlerType
+ * @typedef {import('./types.js').BindParameterConfig} BindParameterConfig
+ */
+
+const DEFAULT_EXCLUSION_LIST = ['sys.', 'dbms_', 'utl_', 'owa_', 'htp.', 'htf.', 'wpg_docload.', 'ctxsys.', 'mdsys.'];
+
+// NOTE: Consider using a separate cache for each database pool to avoid possible conflicts.
+/**
+ * @typedef {{hitCount: number, valid: boolean}} cacheEntryType
+ */
+/** @type {Map<string, cacheEntryType>} */
+const REQUEST_VALIDATION_FUNCTION_CACHE = new Map();
+const REQUEST_VALIDATION_FUNCTION_CACHE_MAX_COUNT = 10000;
+
+/**
+ * Sanitize the procedure name.
+ *
+ * @param {string} procName - The procedure name.
+ * @param {Connection} databaseConnection - The database connection
+ * @param {configPlSqlHandlerType} options - the options for the middleware.
+ * @returns {Promise<string>} Promise resolving to final procedure name.
+ */
+export const sanitizeProcName = async (procName, databaseConnection, options) => {
+	debug('sanitizeProcName', procName);
+
+	// make lowercase and trim
+	let finalProcName = procName.toLowerCase().trim();
+
+	// remove special characters
+	finalProcName = removeSpecialCharacters(finalProcName);
+
+	// check for default exclusions
+	for (const i of DEFAULT_EXCLUSION_LIST) {
+		if (finalProcName.startsWith(i)) {
+			const error = `Procedure name "${procName}" is in default exclusion list "${DEFAULT_EXCLUSION_LIST.join(',')}"`;
+			debug(error);
+			throw new RequestError(error);
+		}
+	}
+
+	// check for custom exclusions
+	if (options.exclusionList && options.exclusionList.length > 0) {
+		for (const i of options.exclusionList) {
+			if (finalProcName.startsWith(i)) {
+				const error = `Procedure name "${procName}" is in custom exclusion list "${options.exclusionList.join(',')}"`;
+				debug(error);
+				throw new RequestError(error);
+			}
+		}
+	}
+
+	// Check request validation function
+	if (options.requestValidationFunction && options.requestValidationFunction.length > 0) {
+		const valid = await requestValidationFunction(finalProcName, options.requestValidationFunction, databaseConnection);
+		if (!valid) {
+			const error = `Procedure name "${procName}" is not valid according to the request validation function "${options.requestValidationFunction}"`;
+			debug(error);
+			throw new RequestError(error);
+		}
+	}
+
+	return finalProcName;
+};
+
+/**
+ * @param {string} str - string
+ * @returns {string} - string
+ */
+const removeSpecialCharacters = (str) => {
+	if (str === null || str === undefined) {
+		return '';
+	}
+
+	const chars = [];
+
+	for (const c of str) {
+		if ((c >= '0' && c <= '9') || (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || c === '.' || c === '_' || c === '#' || c === '$') {
+			chars.push(c);
+		}
+	}
+
+	return chars.join('');
+};
+
+/**
+ * @param {string} procName - The procedure name.
+ * @param {string} requestValidationFunction -  The request validation function.
+ * @param {Connection} databaseConnection - The database connection
+ * @returns {Promise<boolean>} Promise resolving to final procedure name.
+ */
+const loadRequestValid = async (procName, requestValidationFunction, databaseConnection) => {
+	/** @type {BindParameterConfig} */
+	const bind = {
+		proc: {dir: oracledb.BIND_IN, type: oracledb.STRING, val: procName},
+		valid: {dir: oracledb.BIND_OUT, type: oracledb.NUMBER},
+	};
+
+	const SQL = [
+		'DECLARE',
+		'   l_valid NUMBER := 0;',
+		'BEGIN',
+		`   IF (${requestValidationFunction}(:proc)) THEN`,
+		'      l_valid := 1;',
+		'   END IF;',
+		'   :valid := l_valid;',
+		'END;',
+	].join('\n');
+
+	/** @type {Result} */
+	let result = {};
+	try {
+		result = await databaseConnection.execute(SQL, bind);
+	} catch (err) {
+		debug('result', result);
+		/* istanbul ignore next */
+		const message = `Error when validating procedure name "${procName}"\n${SQL}\n${errorToString(err)}`;
+		/* istanbul ignore next */
+		throw new RequestError(message);
+	}
+
+	try {
+		const data = z.object({valid: z.number()}).strict().parse(result.outBinds);
+		return data.valid === 1;
+	} catch (err) {
+		debug('result', result.outBinds);
+		/* istanbul ignore next */
+		const message = `Internal error when parsing ${result.outBinds}\n${errorToString(err)}`;
+		/* istanbul ignore next */
+		throw new Error(message);
+	}
+};
+
+/**
+ *	Remove the cache entries with the lowest hitCount.
+ *	@param {number} count - Number of entries to remove
+ *	@returns {void}
+ */
+const removeLowestHitCountEntries = (count) => {
+	// Convert cache entries to an array
+	const entries = Array.from(REQUEST_VALIDATION_FUNCTION_CACHE.entries());
+
+	// Sort entries by hitCount in ascending order
+	entries.sort((a, b) => a[1].hitCount - b[1].hitCount);
+
+	// Get the keys of the `count` entries with the lowest hitCount
+	const keysToRemove = entries.slice(0, count).map(([key]) => key);
+
+	// Remove these entries from the cache
+	for (const key of keysToRemove) {
+		REQUEST_VALIDATION_FUNCTION_CACHE.delete(key);
+	}
+};
+
+/**
+ * Request validation function.
+ *
+ * @param {string} procName - The procedure name.
+ * @param {string} requestValidationFunction -  The request validation function.
+ * @param {Connection} databaseConnection - The database connection
+ * @returns {Promise<boolean>} Promise resolving to final procedure name.
+ */
+const requestValidationFunction = async (procName, requestValidationFunction, databaseConnection) => {
+	debug('requestValidationFunction', procName, requestValidationFunction);
+
+	// calculate the key
+	//const key = `${databaseConnection.connectString}_${databaseConnection.user}_${procName.toLowerCase()}`;
+	const key = procName.toLowerCase();
+
+	// lookup in the cache
+	const cacheEntry = REQUEST_VALIDATION_FUNCTION_CACHE.get(key);
+
+	// if we fount the procedure in the cache, we increase the hit cound and return
+	if (cacheEntry) {
+		cacheEntry.hitCount++;
+		if (debug.enabled) {
+			debug(`findArguments: procedure "${procName}" found in cache with "${cacheEntry.hitCount}" hits`);
+		}
+		return cacheEntry.valid;
+	}
+
+	// if the cache is full, we remove the 1000 least used cache entries
+	if (REQUEST_VALIDATION_FUNCTION_CACHE.size > REQUEST_VALIDATION_FUNCTION_CACHE_MAX_COUNT) {
+		if (debug.enabled) {
+			debug(`findArguments: cache is full. size=${REQUEST_VALIDATION_FUNCTION_CACHE.size} max=${REQUEST_VALIDATION_FUNCTION_CACHE_MAX_COUNT}`);
+		}
+		removeLowestHitCountEntries(1000);
+	}
+
+	// load from database
+	if (debug.enabled) {
+		debug(`findArguments: procedure "${procName}" not found in cache and must be loaded`);
+	}
+	const valid = await loadRequestValid(procName, requestValidationFunction, databaseConnection);
+
+	// add to the cache
+	REQUEST_VALIDATION_FUNCTION_CACHE.set(key, {hitCount: 0, valid});
+
+	return valid;
+};

package/src/procedureVariable.js

@@ -0,0 +1,57 @@
+/*
+ *	Invoke the Oracle procedure and return the raw content of the page
+ */
+
+import debugModule from 'debug';
+const debug = debugModule('webplsql:procedureVariable');
+
+import oracledb from 'oracledb';
+import {sanitizeProcName} from './procedureSanitize.js';
+
+/**
+ * @typedef {import('oracledb').Connection} Connection
+ * @typedef {import('oracledb').Result<unknown>} Result
+ * @typedef {import('./types.js').configPlSqlHandlerType} configPlSqlHandlerType
+ * @typedef {import('./types.js').argObjType} argObjType
+ * @typedef {import('./types.js').BindParameterConfig} BindParameterConfig
+ */
+
+/**
+ * Get the sql statement and bindings for the procedure to execute for a variable number of arguments
+ * @param {string} procName - The procedure to execute
+ * @param {argObjType} argObj - The arguments to pass to the procedure
+ * @param {Connection} databaseConnection - The database connection
+ * @param {configPlSqlHandlerType} options - the options for the middleware.
+ * @returns {Promise<{sql: string; bind: BindParameterConfig}>} - The SQL statement and bindings for the procedure to execute
+ */
+export const getProcedureVariable = async (procName, argObj, databaseConnection, options) => {
+	if (debug.enabled) {
+		debug(`getProcedureVariable: ${procName} arguments=`, argObj);
+	}
+
+	const names = [];
+	const values = [];
+
+	for (const key in argObj) {
+		const value = argObj[key];
+		if (typeof value === 'string') {
+			names.push(key);
+			values.push(value);
+		} else if (Array.isArray(value)) {
+			value.forEach((item) => {
+				names.push(key);
+				values.push(item);
+			});
+		}
+	}
+
+	const sanitizedProcName = await sanitizeProcName(procName, databaseConnection, options);
+
+	return {
+		sql: `${sanitizedProcName}(:argnames, :argvalues);`,
+		bind: {
+			argnames: {dir: oracledb.BIND_IN, type: oracledb.STRING, val: names},
+			argvalues: {dir: oracledb.BIND_IN, type: oracledb.STRING, val: values},
+		},
+	};
+};

package/src/request.js

@@ -0,0 +1,103 @@
+/*
+ *	Process the http request
+ */
+
+import debugModule from 'debug';
+const debug = debugModule('webplsql:request');
+
+import util from 'node:util';
+import {invokeProcedure} from './procedure.js';
+import {getCGI} from './cgi.js';
+import {getFiles} from './upload.js';
+import {RequestError} from './requestError.js';
+
+/**
+ * @typedef {import('express').Request} Request
+ * @typedef {import('express').Response} Response
+ * @typedef {import('oracledb').Pool} Pool
+ * @typedef {import('oracledb').Connection} Connection
+ * @typedef {import('./types.js').argObjType} argObjType
+ * @typedef {import('./types.js').configPlSqlHandlerType} configPlSqlHandlerType
+ */
+
+/**
+ * Execute the request
+ *
+ * @param {Request} req - The req object represents the HTTP request.
+ * @param {Response} res - The res object represents the HTTP response that an Express app sends when it gets an HTTP request.
+ * @param {configPlSqlHandlerType} options - the options for the middleware.
+ * @param {Pool} connectionPool - The connection pool.
+ * @returns {Promise<void>} - Promise resolving to th page
+ */
+export const processRequest = async (req, res, options, connectionPool) => {
+	debug('executeRequest: ENTER');
+
+	// open database connection
+	const connection = await connectionPool.getConnection();
+
+	// Get the CGI
+	const cgiObj = getCGI(req, options.documentTable ?? '', options.cgi ?? {});
+	debug('executeRequest: cgiObj=', cgiObj);
+
+	// Does the request contain any files
+	const filesToUpload = getFiles(req);
+	debug('executeRequest: filesToUpload=', filesToUpload);
+
+	// Add the query properties
+	/** @type {argObjType} */
+	const argObj = {};
+	Object.assign(argObj, req.query);
+
+	// Add the files to the arguments
+	filesToUpload.reduce((aggregator, file) => {
+		aggregator[file.fieldname] = file.originalname;
+		return aggregator;
+	}, argObj);
+
+	// Does the request contain a body
+	Object.assign(argObj, normalizeBody(req));
+
+	// invoke the Oracle procedure and get the page contenst
+	await invokeProcedure(req, res, argObj, cgiObj, filesToUpload, options, connection);
+
+	// close database connection
+	await connection.release();
+
+	debug('executeRequest: EXIT');
+};
+
+/**
+ *	Is the given value a string or an array of strings
+ *	@param {unknown} value - The value to check.
+ *	@returns {value is string | string[]} - True if the value is a string or an array of strings
+ */
+const isStringOrArrayOfString = (value) => typeof value === 'string' || (Array.isArray(value) && value.every((element) => typeof element === 'string'));
+
+/**
+ *	Normalize the body by making sure that only "simple" parameters and no nested objects are submitted
+ *	@param {Request} req - The req object represents the HTTP request.
+ *	@returns {Record<string, string | string[]>} - The normalized body.
+ */
+const normalizeBody = (req) => {
+	/** @type {Record<string, string | string[]>} */
+	const args = {};
+
+	/* istanbul ignore else */
+	if (typeof req.body === 'object' && req.body !== null) {
+		for (const key in req.body) {
+			// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access
+			const value = req.body[key];
+			/* istanbul ignore else */
+			if (isStringOrArrayOfString(value)) {
+				args[key] = value;
+			} else {
+				/* istanbul ignore next */
+				throw new RequestError(
+					`The element "${key}" in the body is not a string or an array of strings!\n${util.inspect(req.body, {showHidden: false, depth: null, colors: false})}`,
+				);
+			}
+		}
+	}
+
+	return args;
+};

package/src/{requestError.ts → requestError.js}

@@ -1,11 +1,15 @@
 /*
-*	RequestError
-*/
+ *	RequestError
+ */
 
 export class RequestError extends Error {
-	timestamp: Date;
+	/** @type {Date} */
+	timestamp;
 
-	constructor(message: string) {
+	/**
+	 * @param {string} message - The error message.
+	 */
+	constructor(message) {
 		super(message);
 
 		// Maintains proper stack trace for where our error was thrown (only available on V8)

package/src/sendResponse.js

@@ -0,0 +1,104 @@
+/*
+ *	Page the raw page content and return the content to the client
+ */
+
+import debugModule from 'debug';
+const debug = debugModule('webplsql:sendResponse');
+
+import {getBlock} from './trace.js';
+
+/**
+ * @typedef {import('express').Request} Request
+ * @typedef {import('express').Response} Response
+ * @typedef {import('./types.js').pageType} pageType
+ */
+
+/**
+ *	Send "default" response to the browser
+ *	@param {Request} req - The req object represents the HTTP request.
+ *	@param {Response} res - The res object represents the HTTP response that an Express app sends when it gets an HTTP request.
+ *	@param {pageType} page - The page to send.
+ *	@returns {void}
+ */
+export const sendResponse = (req, res, page) => {
+	/** @type {string[]} */
+	const debugText = [];
+
+	// Send the "cookies"
+	page.head.cookies.forEach((cookie) => {
+		const {name, value} = cookie;
+
+		if (debug.enabled) {
+			debugText.push(`res.cookie("${name}", "${value}")`);
+		}
+
+		res.cookie(name, value);
+	});
+
+	// If there is a "redirectLocation" header, we immediately redirect and return
+	if (typeof page.head.redirectLocation === 'string' && page.head.redirectLocation.length > 0) {
+		if (debug.enabled) {
+			debugText.push(`res.redirect(302, "${page.head.redirectLocation}")`);
+			debug(getBlock('RESPONSE', debugText.join('\n')));
+		}
+
+		res.redirect(302, page.head.redirectLocation);
+		return;
+	}
+
+	// Send all the "otherHeaders"
+	for (const key in page.head.otherHeaders) {
+		if (debug.enabled) {
+			debugText.push(`res.set("${key}", "${page.head.otherHeaders[key]}")`);
+		}
+
+		res.set(key, page.head.otherHeaders[key]);
+	}
+
+	// If this is a file download, we eventually set the "Content-Type" and the file content and then return.
+	if (page.file.fileType === 'B' || page.file.fileType === 'F') {
+		if (typeof page.head.contentType === 'string' && page.head.contentType.length > 0) {
+			if (debug.enabled) {
+				debugText.push(`res.writeHead("Content-Type", "${page.head.contentType}")`);
+			}
+
+			res.writeHead(200, {'Content-Type': page.head.contentType});
+		}
+
+		if (debug.enabled) {
+			debugText.push(`res.end("${page.file.fileType}")`);
+			debug(getBlock('RESPONSE', debugText.join('\n')));
+		}
+
+		res.end(page.file.fileBlob, 'binary');
+		return;
+	}
+
+	// Is the a "contentType" header
+	if (typeof page.head.contentType === 'string' && page.head.contentType.length > 0) {
+		if (debug.enabled) {
+			debugText.push(`res.set("Content-Type", "${page.head.contentType}")`);
+		}
+
+		res.set('Content-Type', page.head.contentType);
+	}
+
+	// If we have a "Status" header, we send the header and then return.
+	if (typeof page.head.statusCode === 'number') {
+		if (debug.enabled) {
+			debugText.push(`res.status(page.head.statusCode).send("${page.head.statusDescription}")`);
+			debug(getBlock('RESPONSE', debugText.join('\n')));
+		}
+
+		res.status(page.head.statusCode).send(page.head.statusDescription);
+		return;
+	}
+
+	// Send the body
+	if (debug.enabled) {
+		debugText.push(`${'-'.repeat(60)}\n${page.body}`);
+		debug(getBlock('RESPONSE', debugText.join('\n')));
+	}
+
+	res.send(page.body);
+};

package/src/server.js

@@ -0,0 +1,106 @@
+import debugModule from 'debug';
+const debug = debugModule('webplsql:server');
+
+import express from 'express';
+import bodyParser from 'body-parser';
+import cookieParser from 'cookie-parser';
+import compression from 'compression';
+import {z$configType} from './types.js';
+import {installShutdown} from './shutdown.js';
+import {writeAfterEraseLine} from './tty.js';
+import {poolCreate, poolsClose} from '../src/oracle.js';
+import {handlerUpload} from './handlerUpload.js';
+import {handlerLogger} from './handlerLogger.js';
+import {initMetrics, handlerMetrics} from './handlerMetrics.js';
+import {handlerWebPlSql} from './handlerPlSql.js';
+import {getPackageVersion} from './version.js';
+
+/**
+ * @typedef {import('express').Request} Request
+ * @typedef {import('express').Response} Response
+ * @typedef {import('express').NextFunction} NextFunction
+ * @typedef {import('oracledb').Pool} Pool
+ * @typedef {import('./types.js').environmentType} environmentType
+ * @typedef {import('./types.js').configType} configType
+ */
+
+/**
+ * Start generic server.
+ * @param {configType} config - The config.
+ * @returns {Promise<void>} - Promise.
+ */
+export const startServer = async (config) => {
+	debug('startServer', config);
+
+	config = z$configType.parse(config);
+
+	console.log(`WEB_PL/SQL ${getPackageVersion()}`);
+
+	/** @type {Pool[]} */
+	const pools = [];
+
+	// Install shutdown handler
+	installShutdown(async () => {
+		// Close database pools.
+		await poolsClose(pools);
+		pools.length = 0;
+
+		// Close server
+		return new Promise((resolve) => server.close(() => resolve()));
+	});
+
+	// Create express app
+	const app = express();
+
+	// Metrics
+	const metrics = initMetrics();
+	app.use(handlerMetrics(metrics));
+
+	// Serving static files
+	for (const i of config.routeStatic) {
+		app.use(i.route, express.static(i.directoryPath));
+		console.log(`Static resources on "${i.route}" from directory ${i.directoryPath}`);
+	}
+
+	// Default middleware
+	app.use(handlerUpload());
+	app.use(bodyParser.json());
+	app.use(bodyParser.urlencoded({extended: true}));
+	app.use(cookieParser());
+	app.use(compression());
+
+	// Access log
+	if (config.loggerFilename.length > 0) {
+		console.log(`Access log in "${config.loggerFilename}"`);
+		app.use(handlerLogger(config.loggerFilename));
+	}
+
+	// Oracle pl/sql express middleware
+	for (const i of config.routePlSql) {
+		// Allocate the Oracle database pool
+		const pool = await poolCreate(i.user, i.password, i.connectString);
+		pools.push(pool);
+
+		app.use(`${i.route}/:name?`, handlerWebPlSql(pool, i));
+
+		console.log(`Application route "http://localhost:${config.port}${i.route}"`);
+		console.log(`   Oracle user:           ${i.user}`);
+		console.log(`   Oracle server:         ${i.connectString}`);
+		console.log(`   Oracle document table: ${i.documentTable}`);
+	}
+
+	// Update metrics every second
+	if (config.monitorConsole) {
+		setInterval(() => {
+			// Update requests per second
+			const requestsLastInterval = metrics.requestsInLastInterval;
+			metrics.requestsInLastInterval = 0;
+
+			// Clear console and display metrics
+			writeAfterEraseLine(`Total requests: ${metrics.totalRequests}, requests per second: ${requestsLastInterval}`);
+		}, 1000);
+	}
+
+	// listen on port
+	const server = app.listen(config.port);
+};

package/src/shutdown.js

@@ -0,0 +1,53 @@
+import debugModule from 'debug';
+const debug = debugModule('webplsql:shutdown');
+
+/**
+ * @param {() => Promise<void>} handler - Shutdown handler
+ * @returns {void}
+ */
+const shutdown = (handler) => {
+	handler().then(
+		() => process.exit(0),
+		() => process.exit(1),
+	);
+
+	process.exit(0);
+};
+
+/**
+ * Install a shutdown handler.
+ * @param {() => Promise<void>} handler - Shutdown handler
+ * @returns {void}
+ */
+export const installShutdown = (handler) => {
+	debug('installShutdown');
+
+	/*
+	 *	The 'unhandledRejection' event is emitted whenever a Promise is rejected and no error handler is attached to the promise within a turn of the event loop.
+	 */
+	process.on('unhandledRejection', (reason, p) => {
+		console.log('\nUnhandled promise rejection. Graceful shutdown...', reason, p);
+		shutdown(handler);
+	});
+
+	// install signal event handler
+	process.on('SIGTERM', function sigterm() {
+		console.log('\nGot SIGINT (aka ctrl-c in docker). Graceful shutdown...');
+		shutdown(handler);
+	});
+
+	process.on('SIGINT', function sigint() {
+		console.log('\nGot SIGTERM (aka docker container stop). Graceful shutdown...');
+		shutdown(handler);
+	});
+};
+
+/**
+ * Force a shutdown.
+ * @returns {void}
+ */
+export const forceShutdown = () => {
+	debug('forceShutdown');
+
+	process.kill(process.pid, 'SIGTERM');
+};

package/src/stream.js

@@ -0,0 +1,28 @@
+import stream from 'node:stream';
+
+/**
+ *	Convert a readable stream to a buffer.
+ *
+ * @param {stream.Readable} readable - The readable stream.
+ * @returns {Promise<Buffer>} The buffer.
+ */
+export const streamToBuffer = async (readable) => {
+	return new Promise((resolve, reject) => {
+		/** @type {Buffer[]} */
+		const buffers = [];
+
+		readable.on('data', (buffer) => {
+			// eslint-disable-next-line @typescript-eslint/no-unsafe-argument
+			buffers.push(buffer);
+		});
+
+		readable.on('end', () => {
+			resolve(Buffer.concat(buffers));
+		});
+
+		readable.on('error', (err) => {
+			// istanbul ignore next
+			reject(err);
+		});
+	});
+};