pim-import 3.0.0 → 3.1.0

package/src/algolia/products.ts

@@ -14,7 +14,7 @@ import {
   WrapperImageFields,
 } from "../types";
 import type { Entry } from "contentful-management/dist/typings/entities/entry";
-import { log } from "../libs/logs";
+import { log, serverUtils } from "../libs/logs";
 import { getIndex, AvailableIndicesKey, removeIndexObject } from "./config";
 import {
   getLocalISOTime,
@@ -441,6 +441,7 @@ const getObjects = async (
 
   const objects: AlgoliaProductRecord[] = [];
   let count: number = Number(offset);
+  let current: number = 0;
   for (const topicProduct of items) {
     log(`${++count} of ${total}`, "INFO");
     const timeStart = new Date();
@@ -449,6 +450,11 @@ const getObjects = async (
     const seconds = secondBetweenTwoDate(timeStart, timeEnd);
     log(`Execution time: ${seconds} seconds`);
     objects.push(record);
+    if (serverUtils) {
+      serverUtils.log(topicProduct.sys.id);
+      const progress = Math.floor((++current / items.length) * 100);
+      serverUtils.updateProgress(progress);
+    }
   }
 
   return {

package/src/algolia/subFamilies.ts

@@ -10,7 +10,7 @@ import {
 } from "../libs/contentful-cda";
 import { AlgoliaPaginateRecords, AvailableCatalogs } from "../types";
 import type { Entry } from "contentful-management/dist/typings/entities/entry";
-import { log } from "../libs/logs";
+import { log, serverUtils } from "../libs/logs";
 import { getIndex, AvailableIndicesKey, removeIndexObject } from "./config";
 import {
   getLocalISOTime,
@@ -355,6 +355,7 @@ const getObjects = async (
 
   const objects: AlgoliaFamilyRecord[] = [];
   let count: number = Number(offset);
+  let current: number = 0;
   for (const topicSubFamily of items) {
     log(`${++count} of ${total}`, "INFO");
     const timeStart = new Date();
@@ -363,6 +364,11 @@ const getObjects = async (
     const seconds = secondBetweenTwoDate(timeStart, timeEnd);
     log(`Execution time: ${seconds} seconds`);
     objects.push(record);
+    if (serverUtils) {
+      serverUtils.log(topicSubFamily.sys.id);
+      const progress = Math.floor((++current / items.length) * 100);
+      serverUtils.updateProgress(progress);
+    }
   }
 
   return {

package/src/algolia/subModels.ts

@@ -16,7 +16,7 @@ import {
   TopicProductFieldsResponse,
   WrapperImageFields,
 } from "../types";
-import { log } from "../libs/logs";
+import { log, serverUtils } from "../libs/logs";
 import { getIndex, AvailableIndicesKey, removeIndexObject } from "./config";
 import type { Entry } from "contentful-management/dist/typings/entities/entry";
 import {
@@ -287,6 +287,7 @@ const getObjects = async (
 
   const objects: AlgoliaSubModelRecord[] = [];
   let count: number = Number(offset);
+  let current: number = 0;
   for (const topicSubModel of items) {
     log(`${++count} of ${total}`, "INFO");
     const timeStart = new Date();
@@ -295,6 +296,11 @@ const getObjects = async (
     const seconds = secondBetweenTwoDate(timeStart, timeEnd);
     log(`Execution time: ${seconds} seconds`);
     objects.push(record);
+    if (serverUtils) {
+      serverUtils.log(topicSubModel.sys.id);
+      const progress = Math.floor((++current / items.length) * 100);
+      serverUtils.updateProgress(progress);
+    }
   }
 
   return {

package/src/index.ts

@@ -113,6 +113,7 @@ export { getLocalISOTime } from "./utils";
 export { getStaticDailyProducts, getLatestProducts } from "./pim/endpoints";
 export {
   log,
+  setServerUtils,
   setLogId,
   setLogPath,
   setLogFilename,

package/src/libs/logs.ts

@@ -14,6 +14,7 @@ let logId: string = "";
 let logPath: string = "";
 let logFilename: string = "";
 let Logger: any = null;
+export let serverUtils: any = null;
 
 export const setLogId = (id: string) => {
   if (logId !== id) {
@@ -45,6 +46,10 @@ export const setLogFilename = (filename: string) => {
   }
 };
 
+export const setServerUtils = (utils: any) => {
+  serverUtils = utils;
+};
+
 export const getLogFolder = () => {
   return logPath ? `logs/${logPath}/` : `logs/`;
 };
@@ -81,6 +86,7 @@ export const log = (message: string, level: LogLevels = "INFO") => {
   }
   const winstonLevel: string = level.toLowerCase();
   message = logId ? `[${logId}] - ${message}` : message;
+
   if (level === "ERROR") {
     if (Logger) {
       Logger.log(winstonLevel, message);

package/src/pim/methods/catalogs.ts

@@ -26,7 +26,7 @@ import {
   addFieldValue,
 } from "../../libs/contentful";
 import { getCatalogTaxonomiesHierarchy } from "../endpoints";
-import { log } from "../../libs/logs";
+import { log, serverUtils } from "../../libs/logs";
 import {
   sleep,
   getPimTranslations,
@@ -512,6 +512,7 @@ export const importCategories = async (
   const categoryEntries: string[] = [];
   if (catalogDetails.categories?.length) {
     let count: number = 0;
+    let current: number = 0;
     for (const categoryDetails of catalogDetails.categories) {
       if (offset <= count || limit === -1) {
         log(`${count + 1} of ${catalogDetails.categories.length}`);
@@ -530,6 +531,16 @@ export const importCategories = async (
         }
         await sleep(100);
 
+        if (serverUtils) {
+          serverUtils.log(categoryDetails.data.code);
+          const currentTotal =
+            catalogDetails.categories.length > limit
+              ? limit
+              : catalogDetails.categories.length;
+          const progress = Math.floor((++current / currentTotal) * 100);
+          serverUtils.updateProgress(progress);
+        }
+
         if (limit !== -1 && count + 1 - offset >= limit) {
           break;
         }

package/src/pim/methods/dictionary.ts

@@ -23,7 +23,7 @@ import type {
   CreateEntryProps,
   Entry,
 } from "contentful-management/dist/typings/entities/entry";
-import { log } from "../../libs/logs";
+import { log, serverUtils } from "../../libs/logs";
 import {
   getPimTranslations,
   getLocalISOTime,
@@ -118,6 +118,7 @@ export const importDictionaryIcons = async (
 
     const requiredDictionaryDataWithIcons =
       getRequiredDictionaryDataWithIcons();
+    let current: number = 0;
     for (const reqField of requiredDictionaryDataWithIcons) {
       const reqFieldKey = reqField.key;
       if (reqFieldKey && reqField.icon) {
@@ -143,6 +144,13 @@ export const importDictionaryIcons = async (
           }
         }
       }
+      if (serverUtils) {
+        serverUtils.log(reqFieldKey);
+        const progress = Math.floor(
+          (++current / requiredDictionaryDataWithIcons.length) * 100
+        );
+        serverUtils.updateProgress(progress);
+      }
     }
 
     if (data?.certifications) {
@@ -203,6 +211,7 @@ export const importDictionaryFields = async (
     // Filter required fields
     log("Filter required fields");
     let count: number = 0;
+    let current: number = 0;
     const data: ObjectPartial = {};
     for (const productFieldRequiredData of productFieldsRequiredData) {
       if (offset <= count || limit === -1) {
@@ -231,6 +240,15 @@ export const importDictionaryFields = async (
         }
       }
       count++;
+      if (serverUtils) {
+        serverUtils.log(productFieldRequiredData.key);
+        const currentTotal =
+          productFieldsRequiredData.length > limit
+            ? limit
+            : productFieldsRequiredData.length;
+        const progress = Math.floor((++current / currentTotal) * 100);
+        serverUtils.updateProgress(progress);
+      }
     }
 
     const completed = count >= productFieldsRequiredData.length;

package/src/pim/methods/families.ts

@@ -1,5 +1,5 @@
 import { getFamilies } from "../endpoints";
-import { log } from "../../libs/logs";
+import { log, serverUtils } from "../../libs/logs";
 import type {
   Entry,
   CreateEntryProps,
@@ -260,6 +260,7 @@ export const importFamilies = async (
   const families = await getFamilies();
 
   let count: number = 0;
+  let current: number = 0;
   for (const family of families) {
     if (offset <= count || limit === -1) {
       if (!sanitizeValue(family.data.code)) {
@@ -273,6 +274,13 @@ export const importFamilies = async (
         if (limit !== -1 && count + 1 - offset >= limit) {
           break;
         }
+        if (serverUtils) {
+          serverUtils.log(family.data.code);
+          const currentTotal =
+            families.length > limit ? limit : families.length;
+          const progress = Math.floor((++current / currentTotal) * 100);
+          serverUtils.updateProgress(progress);
+        }
         // Max 7 calls per seconds
         if (count % 2 === 0 && count < families.length) {
           await sleep(1000);

package/src/pim/methods/latestProducts.ts

@@ -1,6 +1,6 @@
 import { AvailableCatalogs, PaginationResults } from "../../types";
 import { getLatestProducts } from "../endpoints";
-import { log } from "../../libs/logs";
+import { log, serverUtils } from "../../libs/logs";
 import { ProductDetails } from "../../resources/ProductDetails";
 import { importProduct } from "./products";
 import { secondBetweenTwoDate } from "../../utils";
@@ -35,11 +35,17 @@ export const importLatestProducts = async (
   log(`${total} products founded`);
 
   let count: number = offset + 1;
+  let current: number = 0;
   for (const product of products) {
     log(`${count} of ${total}`);
     const productDetails: ProductDetails = product.content;
     await importProduct(productDetails, catalog);
     count++;
+    if (serverUtils) {
+      serverUtils.log(productDetails.code);
+      const progress = Math.floor((++current / products.length) * 100);
+      serverUtils.updateProgress(progress);
+    }
   }
 
   const timeEnd = new Date();

package/src/pim/methods/models.ts

@@ -9,7 +9,7 @@ import {
   addFieldValue,
   addToRelationFields,
 } from "../../libs/contentful";
-import { log } from "../../libs/logs";
+import { log, serverUtils } from "../../libs/logs";
 import type {
   Entry,
   CreateEntryProps,
@@ -258,6 +258,7 @@ export const importModels = async (
   if (models?.pagedCollection?.length) {
     log(`Founded ${total} models`);
     let count = offset;
+    let current: number = 0;
     for (const model of models.pagedCollection) {
       log(`${++count} of ${models.totalCount}`);
 
@@ -266,6 +267,14 @@ export const importModels = async (
       await createOrUpdatePage(topicModel);
       await updateComponentModelOfSubFamilyPages(topicModel);
 
+      if (serverUtils) {
+        serverUtils.log(model.code);
+        const progress = Math.floor(
+          (++current / models.pagedCollection.length) * 100
+        );
+        serverUtils.updateProgress(progress);
+      }
+
       // Max 7 calls per seconds
       if (count % 2 === 0 && count < models.totalCount) {
         await sleep(1000);

package/src/pim/methods/products.ts

@@ -13,7 +13,7 @@ import {
   WrapperImageFields,
   CfSys,
 } from "../../types";
-import { log } from "../../libs/logs";
+import { log, serverUtils } from "../../libs/logs";
 import {
   getEntryByCode,
   getAllEntriesByCodes,
@@ -537,12 +537,14 @@ const getProductAssets = async (
       if (
         process.env.FPI_IMGIX_PIM_IMAGE_DOMAIN &&
         pimAsset.url.indexOf(pimDomain) !== -1 &&
-        ["jpg", "svg", "png", "gif"].includes(fileExtension)
+        ["jpg", "png", "gif"].includes(fileExtension)
       ) {
         assetUrl = pimAsset.url.replace(
           pimDomain,
           process.env.FPI_IMGIX_PIM_IMAGE_DOMAIN
         );
+      } else if (["svg"].includes(fileExtension)) {
+        assetUrl = pimAsset.url;
       } else {
         const path = `product-assets/${md5}`;
         const fileName = basename(pimAsset.url);
@@ -1410,14 +1412,24 @@ export const setProductsRelationships = async (
 
   let count: number = 0;
   let updated: number = 0;
+  let current: number = 0;
   for (const product of JSONData) {
     if (offset <= count || limit === -1) {
       log(`${count + 1} of ${JSONData.length}`);
       await setProductRelationships(product.code, fieldKey);
       updated++;
+
+      if (serverUtils) {
+        serverUtils.log(product.code);
+        const currentTotal = JSONData.length > limit ? limit : JSONData.length;
+        const progress = Math.floor((++current / currentTotal) * 100);
+        serverUtils.updateProgress(progress);
+      }
+
       if (updated % 7 === 0) {
         await sleep(500);
       }
+
       if (limit !== -1 && count + 1 - offset >= limit) {
         break;
       }
@@ -1600,6 +1612,7 @@ export const audit = async (
     log(`Founded ${entriesPage.length} topicProduct pages`);
 
     let count: number = offset;
+    let current: number = 0;
     for (const audit of allAudit) {
       log(`${++count} of ${total}`);
       log(`I process the product ${audit.product} with status ${audit.what}`);
@@ -1758,6 +1771,12 @@ export const audit = async (
           `It has not yet been defined how to process the state ${audit.what}`
         );
       }
+
+      if (serverUtils) {
+        serverUtils.log(audit.product);
+        const progress = Math.floor((++current / allAudit.length) * 100);
+        serverUtils.updateProgress(progress);
+      }
     }
 
     const nextOffset = offset + limit;

package/src/pim/methods/subfamilies.ts

@@ -11,7 +11,7 @@ import {
   addFieldValue,
   addDesignerData,
 } from "../../libs/contentful";
-import { log } from "../../libs/logs";
+import { log, serverUtils } from "../../libs/logs";
 import type {
   Entry,
   CreateEntryProps,
@@ -496,6 +496,7 @@ export const importSubFamilies = async (
   if (subFamilies?.pagedCollection?.length) {
     log(`Import ${total} subFamilies`);
     let count = offset;
+    let current: number = 0;
     for (const subFamily of subFamilies.pagedCollection) {
       log(`${++count} of ${subFamilies.totalCount}`);
 
@@ -515,6 +516,14 @@ export const importSubFamilies = async (
 
       await createOrUpdatePage(topicSubFamily);
 
+      if (serverUtils) {
+        serverUtils.log(subFamily.data.code);
+        const progress = Math.floor(
+          (++current / subFamilies.pagedCollection.length) * 100
+        );
+        serverUtils.updateProgress(progress);
+      }
+
       // Max 7 calls per seconds
       if (count % 2 === 0 && count < subFamilies.totalCount) {
         await sleep(1000);

package/src/pim/methods/submodels.ts

@@ -1,5 +1,5 @@
 import { PaginationResults } from "../../types";
-import { log } from "../../libs/logs";
+import { log, serverUtils } from "../../libs/logs";
 import { getOtherCatalogData } from "../endpoints";
 import {
   CollectionSubModels,
@@ -218,11 +218,20 @@ export const importSubModels = async (
   if (subModels?.pagedCollection?.length) {
     log(`Founded ${total} subModels`);
     let count = offset;
+    let current: number = 0;
     for (const subModel of subModels.pagedCollection) {
       log(`${++count} of ${subModels.totalCount}`);
 
       await importSubModel(subModel);
 
+      if (serverUtils) {
+        serverUtils.log(subModel.code);
+        const progress = Math.floor(
+          (++current / subModels.pagedCollection.length) * 100
+        );
+        serverUtils.updateProgress(progress);
+      }
+
       // Max 7 calls per seconds
       if (count % 2 === 0 && count < subModels.totalCount) {
         await sleep(1000);

package/types/fonts.d.ts

@@ -0,0 +1,2 @@
+export declare module "*.woff";
+export declare module "*.woff2";

package/types/index.d.ts

@@ -25,6 +25,6 @@ export { reindexPost, reindexPosts, removePostObject } from "./algolia/news";
 export { importDownloads } from "./downloads/import";
 export { getLocalISOTime } from "./utils";
 export { getStaticDailyProducts, getLatestProducts } from "./pim/endpoints";
-export { log, setLogId, setLogPath, setLogFilename, getLogFolder, } from "./libs/logs";
+export { log, setServerUtils, setLogId, setLogPath, setLogFilename, getLogFolder, } from "./libs/logs";
 export { removeRecordsByStatus } from "./algolia/clean";
 export { generatePDFByUrl } from "./libs/puppeteer";

package/types/libs/logs.d.ts

@@ -1,9 +1,11 @@
 import winston from "winston";
 export declare type LogLevels = "VERBOSE" | "DEBUG" | "INFO" | "WARN" | "ERROR" | "SILLY" | "HTTP";
 export declare const logStatus = true;
+export declare let serverUtils: any;
 export declare const setLogId: (id: string) => void;
 export declare const setLogPath: (path: string) => void;
 export declare const setLogFilename: (filename: string) => void;
+export declare const setServerUtils: (utils: any) => void;
 export declare const getLogFolder: () => string;
 export declare const getLogger: () => winston.Logger;
 export declare const log: (message: string, level?: LogLevels) => void;