skyeye-svc-common-utils 2.0.0-dev0.2 → 2.0.0-dev0.20

package/src/lib/azure/cognitiveSearch/index.ts

@@ -0,0 +1,7 @@
+export * from './QueryType';
+export * from './SearchClient';
+export * from './SearchFacets';
+export * from './SearchFilter';
+export * from './SearchMode';
+export * from './SearchOption';
+export * from './SearchOrder';

package/src/lib/azure/index.ts

@@ -0,0 +1,7 @@
+export * from './cognitiveSearch';
+export * from './azureBlobStorage';
+export * from './azureKeyVault';
+export * from './azureServiceBusClient';
+export * from './azureServiceBusQueue';
+export * from './azureStorageQueue';
+export * from './azureTableStorage';

package/src/lib/fetch/index.ts

@@ -0,0 +1,2 @@
+export * from './sendRequest';
+export * from './sendRequestWithTimeout';

package/src/lib/fetch/sendRequest.ts

@@ -0,0 +1,10 @@
+import fetch from 'node-fetch';
+
+
+/**
+ * @description wrapper of node-fetch
+ * @param {string} url - request endpoint
+ * @param {any} options - fetch options
+ * @returns {any} response
+ */
+export const sendRequest = async (url: string, options: any) => await fetch(url, options);

package/src/lib/fetch/sendRequestWithTimeout.ts

@@ -0,0 +1,38 @@
+import fetch from 'node-fetch';
+import { AbortController } from '@azure/abort-controller';
+import { thirdPartyRegulation } from '../../utils/appConst';
+
+
+/**
+ * @description to store the current processing request, it should only be called 
+ * by function sendRequestWithTimeout
+ */
+const abortedSet: Set<string> = new Set<string>();
+
+/**
+ * @description send a request which will automatically abort within a given time 
+ * if the request is not response yet
+ * @param {string} url - url of request endpoint
+ * @param {any} options - fetch option
+ * @param {number} abortWithinMs - automatically abort after given millisecond
+ * @returns {any} response
+ */
+export async function sendRequestWithTimeout(
+    url: string,
+    options: any,
+    abortWithinMs: number = thirdPartyRegulation.requestLimitationMsec
+): Promise<any> {
+    const controller = new AbortController();
+    const timeout = setTimeout(() => {
+        abortedSet.add(url);
+        controller.abort();
+    }, abortWithinMs);
+    try {
+        const response = await fetch(url, { signal: controller.signal, ...options});
+        clearTimeout(timeout);
+        return response;
+    } catch (err) {
+        if (abortedSet.has(url) && (err as Error).name === 'AbortError') { abortedSet.delete(url); }
+        throw err;
+    }
+}

package/src/lib/httpClient.ts

@@ -1,36 +1,37 @@
+import fetch, { Response, RequestInit } from 'node-fetch';
 import { UserInfo } from '../interfaces';
 import { getInternalClient, userInfoSimplifier } from '../utils/commonUtils';
 import { generateJWT } from '../utils/commonUtils';
 import {commonAppConst, jsonContentTypeHeaders } from '../utils/appConst';
-const fetch = require('node-fetch');
+import { logger } from '../utils/logger/logger';
 
 export async function httpRequest(method: string, url : string,  data: any = null, headers : any = jsonContentTypeHeaders()): Promise<Response> {
-  let init = {};
-  init = data === null ?  {
-      method: method,
-      cache: "no-cache",
-      headers: headers
-  } : {
-    method: method,
-    cache: "no-cache",
-    headers: headers,
-    body: JSON.stringify(data)
-  };
-
-  return fetch(url, init);
+    const init: RequestInit = {
+        method,
+        headers,
+        body: data === null ? undefined : JSON.stringify(data)
+    }
+    return fetch(url, init);
 }
 
 export async function sendInternalHttpRequest(userInfo:UserInfo, client: string, endpoint: string, data: any = null, method?: string, select?:string[]) {
-  method = method ?? commonAppConst.httpMethod.POST;
-  const internalClient = await getInternalClient(client)
-  const url = internalClient + endpoint;
-  let headers:any = jsonContentTypeHeaders();
-  if(userInfo){
-    userInfo = await userInfoSimplifier(userInfo, select)
-    let token: string = `${commonAppConst.httpHeader.BEARER} ${await generateJWT(userInfo)}`;
-    headers[commonAppConst.httpHeader.AUTHORIZATION] = token;
-  }
-  return httpRequest(method, url, data, headers).then(response => {
-    return response.json()
-  })
+    method = method ?? commonAppConst.httpMethod.POST;
+    const internalClient = await getInternalClient(client)
+    const url = internalClient + endpoint;
+    let headers:any = jsonContentTypeHeaders();
+    if(userInfo){
+        userInfo = await userInfoSimplifier(userInfo, select)
+        let token: string = `${commonAppConst.httpHeader.BEARER} ${await generateJWT(userInfo)}`;
+        headers[commonAppConst.httpHeader.AUTHORIZATION] = token;
+    }
+
+    let response: Response;
+    try {
+        response = await httpRequest(method, url, data, headers);
+        return response.json();
+    }
+    catch (err) {
+        logger.error(`Error while sending internal http request, receiving response: ${response}`);
+        throw new Error(`${err}`);
+    }
 }

package/src/lib/index.ts

@@ -0,0 +1,3 @@
+export * from './azure';
+export * from './fetch';
+export * from './teams';

package/src/lib/teams/broadcastLogMessage.ts

@@ -0,0 +1,70 @@
+import { LogLevel, LogMessage } from './logMessage';
+import { appConfig, loadConfig } from '../../utils/middleware/loadConfig';
+import { sendRequestWithTimeout } from '../fetch';
+
+
+/**
+ * @description broadcast log message to specific Microsoft Teams channels
+ * @param {LogLevel} level - level of the log
+ * @param {string} message - log message
+ * @returns {void} the function is deposable, thus it won't return any of information
+ */
+export async function broadcastLogMessage(level: LogLevel, message: string) {
+    // we only broadcast two levels of log.
+    if (level !== 'error' && level !== 'warn') { return; }
+
+    /**
+     * When the keyVaultName is empty, loadConfig() should be called in order to fetch the value as possible
+     * since we are not using lazy-loading here. It would be better to transform into a lazy-loading mechanism.
+     */
+    if (appConfig === undefined) {
+        try {
+            await loadConfig();
+        } catch (err) {
+            throw err;
+        }
+    }
+
+    const time = new Date().toISOString();
+    const logMessage: LogMessage = {
+        level,
+        message,
+        time,
+        environment: appConfig.environment,
+        appName: appConfig.appName,
+        version: appConfig.version
+    };
+
+    const messageCard: Object = ((): Object => {
+        return {
+            '@type': 'MessageCard',
+            '@context': 'http://schema.org/extensions',
+            'summary': 'Teams broadcasting of Logs',
+            'sections': [{
+                'activityTitle': `# ***Service Error Log - ${logMessage.level}***`,
+                'facts': [
+                    { 'name': 'Time', 'value': logMessage.time },
+                    { 'name': 'Environment', 'value': logMessage.environment },
+                    { 'name': 'Service', 'value': logMessage.appName },
+                    { 'name': 'Version', 'value': logMessage.version }, 
+                    { 'name': 'Message', 'value': logMessage.message }
+                ],
+                'markdown': true
+            }],
+            'potentialAction': [
+                {
+                    '@type': 'OpenUri',
+                    'name': 'View Documentation',
+                    'targets': [ { 'os': 'default', 'uri': '' } ]
+                }
+            ]
+        }
+    })();
+
+    const url = level === 'error' ? 
+        appConfig.telemetry.teamsChannelWebHook.errorChannel
+        :
+        appConfig.telemetry.teamsChannelWebHook.warnChannel;
+
+    await sendRequestWithTimeout(url, { method: 'POST', body: JSON.stringify(messageCard) });
+}

package/src/lib/teams/index.ts

@@ -0,0 +1,2 @@
+export * from './broadcastLogMessage';
+export * from './logMessage';

package/src/lib/teams/logMessage.ts

@@ -0,0 +1,55 @@
+import { Environment } from '../../utils';
+import { z } from 'zod';
+
+
+/**
+ * @description we classified the logs into 4 levels and one supporting level:
+ *  - `verbose`: The most fine-grained information only used in rare cases where 
+ *  you need the full visibility of what is happening in your application 
+ *  and inside the third-party libraries that you use
+ *  - `info`: The standard log level indicating that something happened; 
+ *  the application entered a certain state. It should be purely informative and
+ *  not looking into them on a regular basis
+ *  - `warn`: the log level that indicates that something unexpected happened in
+ *  the application, a problem, or a situation that might disturb one of the
+ *  processes. But that doesn’t mean that the application failed
+ *  - `error`: error, states as `FATAL` in the modern log leveling to alleviate DRY.
+ *  It tells that the application encountered an event or entered a state in which 
+ * one of the crucial business functionality is no longer working
+ *  - `disabled`: supporting enum for disabling the functionality of logger
+ * 
+ * @example
+ * ```markdown
+ *  - verbose: annotate each step in the algorithm or each individual query 
+ *  with parameters in your code
+ *  - info: request of authorization API which states whether a user sign-on success or not
+ *  - warn`: a parsing error occurred that resulted in a certain document not being processed
+ *  - error: when the application is not able to connect to a crucial data store like a database
+ *  or one of the sub-service is unavailable
+ * ```
+ * @see https://sematext.com/blog/logging-levels/
+ */
+export const zLogLevel = z.enum(['verbose', 'info', 'warn', 'error', 'disabled']);
+
+/**
+ * @description typedef of `zLogLevel`
+ */
+export type LogLevel = z.infer<typeof zLogLevel>;
+ 
+/**
+ * @description definition of the context that a log should contains
+ * @property {LogLevel} level - log level
+ * @property {string} time - when does the log be generated
+ * @property {string} appName - the service which generates the logs
+ * @property {Environment} environment - log happened in which environment
+ * @property {string} message - log message
+ * @property {string} version - current app version
+ */
+export type LogMessage = {
+    level: LogLevel;
+    time: string;
+    appName: string;
+    environment: Environment;
+    message: string;
+    version: string;
+}

package/src/utils/appConfig.ts

@@ -1,5 +1,5 @@
 import dotenv from 'dotenv';
-import { commonAppConst } from './appConst';
+
 if (process.env.NODE_ENV !== undefined) {
     dotenv.config({ path: `${process.env.NODE_ENV}.env` });
 }
@@ -49,4 +49,4 @@ export const commonAppConfig = {
     DailyDigest: process.env.DAILYDIGEST_CLIENT,
     Announcement: process.env.ANNOUNCEMENT_CLIENT,
     APIM: process.env.APIM_CLIENT,
-}
+}

package/src/utils/appConst.ts

@@ -1,3 +1,5 @@
+import { z } from 'zod';
+
 export const commonAppConst = {
 	emptyGuid: '00000000-0000-0000-0000-000000000000',
 	sampleStatus: {
@@ -155,3 +157,65 @@ export const exitErrorMessage:Record<string, string> = {
 	SQLQueryRunner: `Requests can only be made in the LoggedIn state, not the SentClientRequest state`,
 	UnknownTimeout: `operation timed out for an unknown reason`
 }
+
+/**
+ * @description environment cuts some of the functionalities, including log levels
+ * and local-env file location, to optimize our services.
+ * @enum {string}
+ */
+export const zEnvironment = z.enum(['test', 'dev', 'development', 'preview', 'staging', 'beta', 'production']);
+
+/**
+ * @description typedef of `zEnvironment`
+ */
+export type Environment = z.infer<typeof zEnvironment>;
+
+/**
+ * @description const to regulate, control the third parties' usage
+ * @property {number} requestLimitationMsec - default request abort time
+ * @property {number} requestDefaultRetry - default retry times if the request is failed
+ */
+export const thirdPartyRegulation = {
+    requestLimitationMsec: 5000,
+    requestDefaultRetry: 2
+};
+
+/**
+ * @description settings for logger
+ * @property {string} path - local storage path of log files
+ * @property {boolean} teamsChannelDisabled - when the log level is `warn`/`error`, 
+ * the messages will be delivered to the Teams channel
+ */
+export const loggerSettings = {
+    path: './log/',
+    teamsChannelDisabled: true
+};
+
+/**
+ * @description Index of Azure Cognitive Search
+ * @enum {string}
+ */
+export enum AzureSearchIndex {
+    news = 'news',
+    company = 'company',
+    market = 'market',
+    financial = 'financial',
+    ott = 'ott',
+    nerl = 'nerl',
+    people = 'people',
+    relationship = 'relationship'
+};
+
+/**
+ * @description configuration of Azure Cognitive Search
+ * @property {number} queryRetryDefault - default retry times for query
+ * @property {number} retryIntervalSecond - retry after waiting given second
+ * @property {string} credentialSecretName - credential secret name saved in Azure Key Vault
+ * @property {string} indexSecretName - index secret name saved in Azure Key Vault
+ */
+export const AzureSearchConfig = {
+    queryRetryDefault: 3,
+    retryIntervalSecond: 1,
+    credentialSecretName: 'SkyEye-SVC-Search-Credential',
+    indexSecretName: 'SkyEye-SVC-Search-Index'
+};

package/src/utils/baseClass/appDataSource.ts

@@ -0,0 +1,23 @@
+import { DataSource, DataSourceOptions } from 'typeorm';
+import { logger } from '../logger/logger';
+
+export class AppDataSource {
+
+    private static instance: DataSource;
+
+    private constructor() {}
+
+    public static getInstance() {
+        return AppDataSource.instance;
+    }
+
+    public static async startConnection(options: DataSourceOptions) {
+        if (!AppDataSource.instance) {
+            AppDataSource.instance = new DataSource(options);
+        }
+
+        if (!AppDataSource.instance.isInitialized) {
+            await AppDataSource.instance.initialize();
+        }
+    }
+}

package/src/utils/baseClass/baseController.ts

@@ -7,7 +7,7 @@ import { QueryRunner } from 'typeorm';
 import { commonAppConst } from '../appConst';
 import { commonAppConfig } from '../appConfig';
 import { Mutex } from 'async-mutex';
-import { getDataSource } from './dataSourceManager';
+import { AppDataSource } from './appDataSource';
 
 
 export abstract class BaseController {
@@ -18,8 +18,8 @@ export abstract class BaseController {
         let queryRunner: QueryRunner;
 
         try {
-            const connection = getDataSource();
-            queryRunner = connection.createQueryRunner();
+            // Use queryRunner if transaction is needed
+            queryRunner = AppDataSource.getInstance().createQueryRunner();
             await queryRunner.startTransaction();
 
             return Promise.resolve(queryRunner);
@@ -59,12 +59,12 @@ export abstract class BaseController {
         } catch (err) {
             logger.error(`BaseController/endTransaction: ${err}`);
         } finally {
-            if (!baseControllerParameter.queryRunner.isReleased) {
-                try {
-                    await baseControllerParameter.queryRunner.release();
-                } catch (err) {
-                    logger.error(`BaseController/endTransaction Release Error: ${err}`);
+            try {
+                if (!baseControllerParameter.queryRunner.isReleased) {
+                        await baseControllerParameter.queryRunner.release();
                 }
+            } catch (err) {
+                logger.error(`BaseController/endTransaction Release Error: ${err}`);
             }
         }
     }
@@ -149,7 +149,7 @@ export abstract class BaseController {
                     baseControllerParameter.serviceResponse.isSuccess = false;
                     baseControllerParameter.serviceResponse.error = true;
                     baseControllerParameter.serviceResponse.message = err.message;
-
+                    
                     throw err;
                 });
             }

package/src/utils/index.ts

@@ -0,0 +1,2 @@
+export * from './appConfig';
+export * from './appConst';

package/src/utils/logger/morganLogger.ts

@@ -7,7 +7,7 @@ import dateformat from 'dateformat';
 // ensure log directory exists
 fs.existsSync(commonAppConfig.LogPath) || fs.mkdirSync(commonAppConfig.LogPath)
  
-function FileNameGenerator(time:Date, index:number){
+const FileNameGenerator = (time: number | Date, index: number) =>{
     if (!time)
     {
         time = new Date();
@@ -18,7 +18,7 @@ function FileNameGenerator(time:Date, index:number){
 }
 
 // create a rotating write stream
-var accessLogStream = rfs(FileNameGenerator, {
+var accessLogStream = rfs.createStream(FileNameGenerator, {
   interval: '1d', // rotate daily
   path: commonAppConfig.LogPath
 })

package/src/utils/middleware/loadConfig.ts

@@ -0,0 +1,150 @@
+import dotenv from 'dotenv';
+import { getKeyVault } from '../../lib';
+import { logger } from '../logger/logger';
+import { z } from 'zod';
+import { zEnvironment, Environment } from '../appConst';
+
+/**
+ * @description
+ * @property {string} appName - service's app name from package.json
+ * @property {string} version - service's version from package.json
+ * @property {enum} environment - current environment
+ * @property {string} keyVaultName - Azure key vault name
+ * @property {string} keyVaultSecretName - secret name of Azure key vault
+ * @todo migration to single typedef
+ */
+const zEnvFileAppConfig = z.object({
+    appName: z.string(),
+    version: z.string(),
+    environment: zEnvironment,
+    keyVaultName: z.string(),
+    keyVaultSecretName: z.string()
+}).strict();
+
+/**
+ * @description typedef of environment File App Config
+ */
+type EnvFileAppConfig = z.infer<typeof zEnvFileAppConfig>;
+
+/**
+ * @todo migration to single typedef
+ */
+const zElasticKeyVault = z.object({
+    secretToken: z.string(),
+    serverUrl: z.string()
+})
+
+/**
+ * @todo migration to single typedef
+ */
+const zTeamsChannelWebHook = z.object({
+    errorChannel: z.string(),
+    warnChannel: z.string()
+})
+
+/**
+ * @todo migration to single typedef
+ */
+const zTelemetry = z.object({
+    teamsChannelWebHook: zTeamsChannelWebHook,
+    elasticKeyVault: zElasticKeyVault
+})
+
+/**
+ * @todo migration to single typedef
+ */
+ const zSecretFileAppConfig = z.object({
+    telemetry: zTelemetry,
+    serviceConfig: z.any().nullable()
+});
+
+/**
+ * @description typedef of secret file
+ */
+type SecretFileAppConfig = z.infer<typeof zSecretFileAppConfig>;
+
+/**
+ * @todo migration to single typedef
+ */
+type AppConfig = EnvFileAppConfig & SecretFileAppConfig;
+
+/**
+ * @description configuration carrier (or, instance). We can classified all the configurations into 2 categories:
+ *  - Environment File App Config: configuration from the environment file (.env) and some other settings
+ *  - Secret File App Config: loaded from the Azure Key Vault
+ */
+export let appConfig: AppConfig;
+
+/**
+ * @todo move to loadConfig since all the environment fields should be loaded from appConfig
+ */
+export const environment: Environment = (process.env.NODE_ENV as Environment) ?? 'development';
+
+/**
+ * @description preload for app configuration, once it is been called, the appConfig will be loaded and be freezed
+ * @param packagePath - optional, path of `package.json` in order to load app name and version
+ */
+export const loadConfig = async (packagePath?: string) => {
+    logger.verbose('start /appConfig/preload()');
+
+    // load environment file
+    dotenv.config({ path: `${environment}.env` });
+
+    /**
+     * To load the appName and the version, we read the package information in the following order:
+     * 1. check if the environment value is defined directly
+     * 2. check if environment given from npm is defined
+     * 3. check if value in package.json is defined
+     */
+    const pkg = require(packagePath ?? `${process.env.PWD}/package.json`);
+
+    const appName: string = ((): string => {
+        if (!!process.env.APP_NAME) { return process.env.APP_NAME; }
+        if (!!process.env.npm_package_name) { return process.env.npm_package_name; }
+        if (!!pkg.name) { return pkg.name; }
+        return 'default-app-name';
+    })();
+    
+    const version: string = ((): string => {
+        if (!!process.env.VERSION) { return process.env.VERSION; }
+        if (!!process.env.npm_package_version) { return process.env.npm_package_version; }
+        if (!!pkg.version) { return pkg.version; }
+        return '0.0.0';
+    })();
+
+    /**
+     * check the env which suppose to be hide as possible as it can, 
+     * which is not allowed to define in-memory.
+     */
+    if (process.env.AZURE_TENANT_ID === undefined || process.env.AZURE_CLIENT_ID === undefined || process.env.AZURE_CLIENT_SECRET === undefined) {
+        throw new Error('/appConfig/preload: missing Azure configuration');
+    }
+
+    const envFileAppConfig: EnvFileAppConfig = {
+        appName,
+        version,
+        environment,
+        keyVaultName: process.env.KEYVAULT_NAME,
+        keyVaultSecretName: process.env.LIBRARY_KEY_VAULT_SECRET_NAME
+    };
+
+    // check envFileAppConfig
+    const envTypeCheck = zEnvFileAppConfig.safeParse(envFileAppConfig);
+    if (!envTypeCheck.success) {
+        throw new Error('/appConfig/preload: invalid schema of envFileAppConfig');
+    }
+
+    // check secretFileAppConfig
+    const rawSecretResp = await getKeyVault(envFileAppConfig.keyVaultName, envFileAppConfig.keyVaultSecretName);
+    if (!rawSecretResp.isSuccess) { throw new Error(rawSecretResp.message); }
+    const secretFileAppConfig: SecretFileAppConfig = rawSecretResp.data;
+    const secretFileTypeCheck = zSecretFileAppConfig.safeParse(secretFileAppConfig);
+    if (!secretFileTypeCheck.success) {
+        throw new Error('/appConfig/preload: invalid schema of secretFileAppConfig');
+    }
+
+    appConfig = { ...envFileAppConfig, ...secretFileAppConfig };
+    Object.freeze(appConfig);
+
+    logger.verbose('end /appConfig/preload()');
+};

package/.env

@@ -1,3 +0,0 @@
-PUBLIC_JWT_SECRET=unittest
-USER_PUBLIC_RETRIEVE = usersvc
-USER_INTERNAL_RETRIEVE = usersvc

package/dist/index.d.ts

@@ -1,29 +0,0 @@
-export * from './utils/appConfig';
-export * from './utils/appConst';
-export * from './utils/commonUtils';
-export * from './utils/authentication/auth';
-export * from './utils/logger/logger';
-export * from './utils/logger/morganLogger';
-export * from './utils/middleware/handleError';
-export * from './utils/middleware/isPublicUser';
-export * from './utils/middleware/overwriteConst';
-export * from './utils/middleware/startdbConnection';
-export * from './utils/baseClass/baseController';
-export * from './utils/baseClass/baseService';
-export * from './interfaces/index';
-export * from './lib/azure/azureKeyVault';
-export * from './lib/azure/azureServiceBusClient';
-export * from './lib/azure/azureStorageQueue';
-export * from './lib/azure/azureServiceBusQueue';
-export * from './lib/azure/azureTableStorage';
-export * from './lib/azure/azureBlobStorage';
-export * from './utils/documentation/swagger';
-export * from './lib/httpClient';
-export * from './utils/monitoring/apm';
-export * from './models/baseControllerParameter';
-export * from './models';
-export * from './lib/redisClient';
-export * from './lib/sendGridClient';
-export * from './interfaces/grpcServiceResponse';
-export * from './interfaces/grpcListRequest';
-export * from './utils/commonUtils/appConfigKeyRetriever';