skyeye-svc-common-utils 0.0.181 → 0.0.185

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "skyeye-svc-common-utils",
-  "version": "0.0.181",
+  "version": "0.0.185",
   "description": "",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -21,8 +21,10 @@
   },
   "homepage": "https://github.com/HandshakesByDC/SkyEye-SVC-Common-Utils#readme",
   "dependencies": {
+    "@azure/functions": "^3.0.0",
     "@azure/identity": "latest",
     "@azure/keyvault-secrets": "latest",
+    "@azure/search-documents": "^11.2.1",
     "@azure/service-bus": "^7.1.0",
     "@azure/storage-blob": "^12.1.2",
     "@azure/storage-queue": "^12.0.2",
@@ -49,7 +51,7 @@
     "node-fetch": "^2.6.0",
     "nyc": "^15.1.0",
     "rotating-file-stream": "^1.4.6",
-    "skyeye-common-const": "0.0.155",
+    "skyeye-common-const": "0.0.161",
     "swagger-jsdoc": "^3.5.0",
     "swagger-ui-express": "^4.1.3",
     "ts-node": "^9.0.0",

package/src/lib/azureSearch/AzureCognitiveSearchClient.ts

@@ -0,0 +1,127 @@
+import { SearchClient, AzureKeyCredential } from '@azure/search-documents';
+import { Semaphore } from 'async-mutex';
+import { ServiceResponse } from '../../interfaces/serviceResponse';
+import { azureSearchConf, AzureSearchIndex } from '../../utils/appConst';
+import { LoggerModel, ExpressServerLogger } from '../../utils/logger';
+import { getKey } from '../azure/azureKeyVault';
+
+
+/**
+ * AzureCognitiveSearchClient as an adaptor to bind the third-party library
+ * (@azure/search-documents) with the internal application, that is to say, the 
+ * class is aim to enhance dev's maintenance between lib and the exact usage.
+ * @param instance - the class is used as a Singleton
+ * @param endpoint - Azure cognitive search's request URL
+ * @param credential - Azure cognitive search's API key
+ * @param semaphore - to ensure not too many request are sent to Azure cognitive search
+ * @param indexMap - to store the index's alias
+ * 
+ * @example
+ * const client = await AzureCognitiveSearchClient.getInstance();
+ * const searchOption = new SearchOption( { top: 1 });
+ * logger.info(JSON.stringify(await client.query(AzureSearchIndex.company, '*', searchOption.value())));
+ */
+export class AzureCognitiveSearchClient {
+    private static instance: AzureCognitiveSearchClient;
+    private static endpoint: string;
+    private static credential: string;
+    private static semaphore: Semaphore;
+    private static indexMap: Record<AzureSearchIndex, string>;
+    private static logger: LoggerModel;
+
+    /**
+     * To retrieve the instance in the Singleton pattern
+     * @returns THE ONLY ONE instance of AzureCognitiveSearchClient
+     */
+    static async getInstance(): Promise<AzureCognitiveSearchClient> {
+        if (!!AzureCognitiveSearchClient.instance) { return AzureCognitiveSearchClient.instance; }
+        AzureCognitiveSearchClient.instance = new AzureCognitiveSearchClient();
+        const secret: { endpoint: string, credential: string } = JSON.parse(await getKey(azureSearchConf.key));
+        AzureCognitiveSearchClient.indexMap = JSON.parse(await getKey(azureSearchConf.indexKey));
+        AzureCognitiveSearchClient.endpoint = secret.endpoint;
+        AzureCognitiveSearchClient.credential = secret.credential;
+        AzureCognitiveSearchClient.semaphore = new Semaphore(azureSearchConf.semaphoreParallelMaximum);
+        
+        // @default: the default of logger is to use Express Server.
+        AzureCognitiveSearchClient.logger = new ExpressServerLogger();
+        return AzureCognitiveSearchClient.instance;
+    }
+
+    setLogger(logger: LoggerModel) {
+        AzureCognitiveSearchClient.logger = logger;
+    }
+
+    /**
+     * To query the Azure cognitive search
+     * @param index to retrieve the document
+     * @param keyword as the prop 'search'
+     * @param options additional option for the search
+     * @param retryTimes specify how many times if Azure cognitive search returns a bad request
+     * @returns ServiceResponse, noted that data is stated in data.document
+     */
+    async query(
+        index: AzureSearchIndex,
+        keyword: string = '*',
+        options?: any,
+        retryTimes: number = azureSearchConf.queryRetryDefault
+    ): Promise<ServiceResponse> {
+
+        AzureCognitiveSearchClient.logger.info('start AzureCognitiveSearchClient/query');
+        let serviceResponse: ServiceResponse = { isSuccess: false };
+        let searchResults: any;
+        try {
+            let curRetry = 0;
+
+            const client = new SearchClient(
+                AzureCognitiveSearchClient.endpoint,
+                AzureCognitiveSearchClient.indexMap[index],
+                new AzureKeyCredential(AzureCognitiveSearchClient.credential)
+            );
+
+            while (curRetry < retryTimes) {
+                await AzureCognitiveSearchClient.semaphore.runExclusive(async () => {
+                    try {
+                        searchResults = await client.search(keyword, options);
+
+                        serviceResponse.data = { document: [] };
+
+                        for await (const result of searchResults.results) {
+                            serviceResponse.data.document.push(result.document);
+                        }
+
+                        if (!!searchResults.facets) {
+                            serviceResponse.data.facets = searchResults.facets;
+                        }
+
+                        if (!!searchResults.count) {
+                            serviceResponse.data.count = searchResults.count;
+                        }
+
+                        serviceResponse.isSuccess = true;
+                    } catch (err) {
+                        AzureCognitiveSearchClient.logger.error(`AzureCognitiveSearchClient/query: fail - ${err}`);
+                    }
+                });
+
+                if (!serviceResponse.isSuccess) {
+                    AzureCognitiveSearchClient.logger.warn(`AzureCognitiveSearchClient/query: retry - ${curRetry}`);
+                    ++curRetry;
+                    this.sleep(azureSearchConf.retryIntervalSecond * 1000);
+                } else {
+                    break;
+                }
+            }
+        
+        } catch (err) {
+            AzureCognitiveSearchClient.logger.error(`AzureCognitiveSearchClient/query: ${err}`);
+        }
+
+        return serviceResponse;
+    }
+
+    /**
+     * To postpone for specific millisecond
+     * @param ms - time to wait for
+     */
+    private async sleep(ms: number) { await new Promise(_ => setTimeout(_, ms)); }
+}

package/src/lib/azureSearch/SearchFilter.ts

@@ -0,0 +1,126 @@
+/**
+ * SearchFilter is the object to construct the Filter when using Azure cognitive search,
+ * @param attr - the attribute, so called props, to be filtered by the condition
+ * @param condition - the condition to filter, see enum @SearchFilter for more information
+ * @param value - attr should under the condition with value
+ */
+export class SearchFilter {
+    attr: string;
+    condition: SearchFilterCondition;
+    value: any;
+    isCollection: boolean = false;
+    collectionOperation: 'any' | 'all' = 'any';
+
+    constructor(
+        attr: string,
+        condition: SearchFilterCondition,
+        value: any,
+        isCollection: boolean = false,
+        collectionOperation: 'any' | 'all' = 'any') {
+        this.attr = attr;
+        this.condition = condition;
+        this.value = value;
+        this.isCollection = isCollection;
+        this.collectionOperation = collectionOperation;
+    }
+
+    /**
+     * flatten is to compose the instance into a string when using Search
+     * @returns the composite of the instance
+     */
+    flatten(): string {
+        if (this.isCollection) {
+            switch (this.condition) {
+                case CONTAINS:
+                    return `${this.attr}/${this.collectionOperation}(i: search.in(i,'${this.value.toString()}', ','))`;
+                case EXCLUDE:
+                    return `${this.attr}/${this.collectionOperation}(i: not search.in(i,'${this.value.toString()}', ','))`;
+                default:
+                    throw new Error(`SearchFilter/flatten: Unknown condition for collection`);
+            }
+        } else {
+            switch (this.condition) {
+                case EQUAL:
+                case NOT_EQUAL:
+                case LESS_THAN_OR_EQUAL:
+                case LESS_THAN:
+                case GREATER_THAN_OR_EQUAL:
+                case GREATER_THAN:
+                    return `${this.attr} ${this.condition} ${ typeof(this.value) === 'string' ? `'${this.value}'` : this.value }`;
+                case CONTAINS:
+                    return `search.in(${this.attr}, '${this.value.toString()}', ',')`;
+                case EXCLUDE:
+                    return `${this.condition} search.in(${this.attr}, '${this.value.toString()}', ',')`;
+                default:
+                    throw new Error(`SearchFilter/flatten: Unknown condition`);
+            }
+        }
+    }
+}
+
+/**
+ * SearchFilterCondition is equally to the condition when filtering an params.
+ */
+export enum SearchFilterCondition {
+    contains = '',
+    exclude = 'not',
+    equal = 'eq',
+    notEqual = 'ne',
+    lessThanOrEqual = 'le',
+    lessThan = 'lt',
+    greaterThanOrEqual = 'ge',
+    greaterThan = 'gt'
+}
+
+/**
+ * The following const params is a easy-to-use type when implementing a SearchFilter,
+ * so that the developers no longer need to call the enum with the class reference
+ */
+export const CONTAINS = SearchFilterCondition.contains;
+export const EXCLUDE = SearchFilterCondition.exclude;
+export const EQUAL = SearchFilterCondition.equal;
+export const NOT_EQUAL = SearchFilterCondition.notEqual;
+export const LESS_THAN_OR_EQUAL = SearchFilterCondition.lessThanOrEqual;
+export const GREATER_THAN_OR_EQUAL = SearchFilterCondition.greaterThanOrEqual;
+export const GREATER_THAN = SearchFilterCondition.greaterThan;
+export const LESS_THAN = SearchFilterCondition.lessThan;
+
+/**
+ * combination is a function ONLY be used for constructing two function: AND and OR,
+ * then it would be express into a string.
+ * @param type - to indicate the combination type
+ * @param args - the arguments which need to be construct into string
+ */
+const combination = (type: 'and' | 'or', ...args: Array<string | SearchFilter>): string => {
+    if (args.length === 0) { return ''; }
+     
+    function flatten(obj: string | SearchFilter): string {
+        return typeof(obj) === 'string' ? obj : obj.flatten();
+    }
+    
+    let res: string = !!args[0] ? flatten(args[0]) : '';
+
+    for (let i = 1; i < args.length; ++i) {
+        if (!!args) {
+            res += ` ${type} ${flatten(args[i])}`;
+        }
+    }
+
+    return `(${res})`;
+}
+
+/**
+ * AND helps developers to compose a nested filter for condition AND.
+ * @param args - the arguments which need to be construct into string
+ */
+export const AND = (...args: Array<string | SearchFilter>): string => {
+    return combination('and', ...args);
+}
+
+/**
+ * OR helps developers to compose a nested filter for condition OR.
+ * @param args - the arguments which need to be construct into string
+ */
+export const OR = (...args: Array<string | SearchFilter>): string => {
+    return combination('or', ...args);
+}

package/src/lib/azureSearch/SearchOption.ts

@@ -0,0 +1,148 @@
+import { AND, SearchFilter } from "./SearchFilter";
+
+
+/**
+ * SearchOptionType is the interface to connect with the library @azure/search-documents,
+ * it would only be initiated when creating an instance of class @SearchOption,
+ * none of the other functions should access this type declaration.
+ * @param select - optional, to indicate the select attributes/props.
+ * @param filter - optional, search with the filter
+ * @param orderBy - optional, return order
+ * @param count - optional, if true, return with the count of the documents.
+ * @param top - optional, total volume when retrieving the data
+ * @param skip - optional, skip how many document
+ * @param facets - optional, retrieve data's facets
+ * @param searchMode - required, different mode for search
+ */
+type SearchOptionType = {
+    select?: string[],
+    filter?: string | Array<string | SearchFilter>,
+    orderBy?: SearchOrder[],
+    count?: boolean,
+    top?: number,
+    skip?: number,
+    facets?: SearchFacets[],
+    searchMode?: SearchMode
+}
+/**
+ * @classdesc the class is to wrap an option when retrieving the data from
+ * @azure/search-documents/query
+ * @param searchOptionType - to store the interface of the option, noted that
+ * when calling the query, you should use SearchOption.value() instead of using
+ * this attribute.
+ */
+export class SearchOption {
+    searchOptionType?: SearchOptionType;
+    
+    constructor(searchOptionType?: SearchOptionType) {
+        this.searchOptionType = searchOptionType ?? {
+            count: false,
+            top: 50,
+            skip: 0,
+            searchMode: SearchMode.all
+        }
+    }
+
+    /**
+     * to compose flatten-filter string for @azure/search-documents
+     * @returns flatten-filter string
+     */
+    private composeFilter(): string {
+        if (typeof(this.searchOptionType.filter) === 'string') {
+            return this.searchOptionType.filter;
+        }
+        return AND(...this.searchOptionType.filter);
+    }
+
+    /**
+     * to compose flatten-order string for @azure/search-documents
+     * @returns flatten-order string
+     */
+    private composeOrder(): string[] {
+        return this.searchOptionType.orderBy.map(
+            (s: SearchOrder) => `${s.attr} ${s.orderBy}`
+        );
+    }
+
+    /**
+     * to compose flatten-facets string for @azure/search-documents
+     * @returns flatten-facets string
+     */
+    private composeFacets(): string[] {
+        return this.searchOptionType.facets.map((s: SearchFacets) => {
+            let res = `${s.attr},count:${s.count}`;
+            if (!!s.sort) { res += `,sort:${s.sort}`; }
+            return res;
+        });
+    }
+
+    /**
+     * as an adaptor to transform the type for @azure/search-documents
+     * @returns object which can be accepted by @azure/search-documents
+     */
+    value = () => {
+        let value: {
+            select?: string[],
+            filter?: string;
+            orderBy?: string[],
+            includeTotalCount?: boolean,
+            top?: number,
+            skip?: number,
+            facets?: string[],
+            searchMode?: string,
+        } = {};
+
+        value.includeTotalCount = this.searchOptionType.count;
+        value.select = this.searchOptionType.select;
+        value.skip = this.searchOptionType.skip;
+        value.top = this.searchOptionType.top;
+        value.searchMode = this.searchOptionType.searchMode;
+        
+        if (!!this.searchOptionType.filter) {
+            value.filter = this.composeFilter();
+        }
+
+        if (!!this.searchOptionType.select && this.searchOptionType.select.length) {
+            value.select = this.searchOptionType.select;
+        }
+
+        if (!!this.searchOptionType.orderBy && this.searchOptionType.orderBy.length) {
+            value.orderBy = this.composeOrder();
+        }
+    
+        if (!!this.searchOptionType.facets && this.searchOptionType.facets.length) {
+            value.facets = this.composeFacets();
+        }
+
+        return value;
+    }
+}
+
+/**
+ * SearchOrder as a type to depict the orderBy props
+ * @params attr - the attribute/prop regards to be the target
+ * @params orderBy - descent or ascent
+ */
+export type SearchOrder = {
+    attr: string;
+    orderBy: 'desc' | 'asc';
+}
+
+/**
+ * SearchMode is defined by Azure cognitive search
+ */
+export enum SearchMode {
+    all = 'all'
+}
+
+/**
+ * SearchFacets as a type is to wrap facets for @azure/search-documents
+ * @param attr - attribute/prop to reach
+ * @param count - volume of the search
+ * @param sort - order
+ */
+export type SearchFacets = {
+    attr: string;
+    count: number;
+    sort?: 'count' | '-count' | 'value' | '-value';
+}

package/src/lib/azureSearch/index.ts

@@ -0,0 +1,3 @@
+export * from './AzureCognitiveSearchClient';
+export * from './SearchFilter';
+export * from './SearchOption';

package/src/utils/appConst.ts

@@ -112,4 +112,25 @@ export enum CAMPAIGN_USER_STATUS {
 export const exitErrorMessage:Record<string, string> = {
 	SQLQueryRunner: `Requests can only be made in the LoggedIn state, not the SentClientRequest state`,
 	UnknownTimeout: `operation timed out for an unknown reason`
+}
+
+export const azureSearchConf = {
+    queryRetryDefault: 3,
+    semaphoreParallelMaximum: 5,
+    retryIntervalSecond: 1,
+    key: 'SkyEye-SVC-Search-Credential',
+    indexKey: 'SkyEye-SVC-Search-Index'
+}
+
+export enum AzureSearchIndex {
+    news = 'news',
+    company = 'company',
+    market = 'market',
+    financial = 'financial',
+    ott = 'ott',
+    nerl = 'nerl',
+    newNerl = 'newNerl',
+    pocNews = 'azure-search-content-poc',
+    pocNerl = 'azure-search-nerl-poc',
+    pocNewsDuplicate = 'azure-search-contentduplicate-poc'
 }

package/src/utils/logger/AzureFunctionLogger.ts

@@ -0,0 +1,27 @@
+import { LoggerModel } from '.';
+import { Context } from '@azure/functions';
+
+
+/**
+ * @todo document
+ */
+export class AzureFunctionLogger extends LoggerModel {
+    context: Context;
+    
+    constructor(context?: Context) {
+        super();
+        this.context = context;
+    }
+
+    info(content: any) {
+        !!content ? this.context.log.info(`${content}`) : console.log(`[info] ${content}`);
+    }
+
+    error(content: any) {
+        !!content ? this.context.log.error(`${content}`) : console.log(`[error] ${content}`);
+    }
+
+    warn(content: any) {
+        !!content ? this.context.log.warn(`${content}`) : console.log(`[warn] ${content}`);
+    }
+}

package/src/utils/logger/ExpressServerLogger.ts

@@ -0,0 +1,18 @@
+import winston from 'winston';
+import { LoggerModel } from '.';
+import { logger as defaultLogger } from '../../utils/logger/logger';
+
+
+/**
+ * @todo document
+ */
+export class ExpressServerLogger extends LoggerModel {
+    logger: winston.Logger;
+    constructor(logger: winston.Logger = defaultLogger) {
+        super();
+        this.logger = logger;
+    }
+    info(content: any) { this.logger.info(content); }
+    error(content: any) { this.logger.error(content); }
+    warn(content: any) { this.logger.warn(content); }
+}

package/src/utils/logger/LoggerModel.ts

@@ -0,0 +1,8 @@
+/**
+ * @todo document
+ */
+export abstract class LoggerModel {
+    abstract info(content: any): void | Promise<void>;
+    abstract error(content: any): void | Promise<void>;
+    abstract warn(content: any): void | Promise<void>;
+}

package/src/utils/logger/index.ts

@@ -0,0 +1,3 @@
+export * from './LoggerModel';
+export * from './AzureFunctionLogger';
+export * from './ExpressServerLogger';

package/src/utils/logger/logger.ts

@@ -22,9 +22,10 @@ export const logger = createLogger({
 
 // If we're not in production then **ALSO** log to the `console`
 // with the colorized simple format.
-if (commonAppConfig.NodeEnv === 'dev') {
+if (commonAppConfig.NodeEnv !== 'production') {
   logger.add(new transports.Console({
     format: format.combine(
+      format.colorize(),
       format.simple()
     )
   }));