sr-npm 1.7.118 → 1.7.119

package/backend/consts.js

@@ -1,4 +1,4 @@
-const {saveJobsDataToCMS,saveJobsDescriptionsAndLocationApplyUrlToCMS,aggregateJobsByFieldToCMS,referenceJobsToField,createApiKeyCollectionAndFillIt} = require('./data');
+const {saveJobsDataToCMS,saveJobsDescriptionsAndLocationApplyUrlToCMS,aggregateJobsByFieldToCMS,referenceJobsToField,createApiKeyCollectionAndFillIt,createCollections,aggregateJobs,referenceJobs} = require('./data');
 const { createCollectionIfMissing } = require('@hisense-staging/velo-npm/backend');
 const { COLLECTIONS, COLLECTIONS_FIELDS, JOBS_COLLECTION_FIELDS } = require('./collectionConsts');
 
@@ -8,13 +8,9 @@ const TASKS_NAMES = {
     SYNC_JOBS: 'syncJobsFromSRAPIToCMS',
     INSERT_JOBS_TO_CMS: 'insertJobsToCMS',
     INSERT_JOBS_DESCRIPTIONS_LOCATION_APPLY_URL_TO_CMS: 'insertJobsDescriptionsLocationApplyUrlToCMS',
-    FILL_JOBS_PER_CITY_COLLECTION: 'fillJobsPerCityCollection',
-    FILL_JOBS_PER_DEPARTMENT_COLLECTION: 'fillJobsPerDepartmentCollection',
-    REFERENCE_JOBS_TO_LOCATIONS: 'referenceJobsToLocations',
-    REFERENCE_JOBS_TO_DEPARTMENT: 'referenceJobsToDepartment',
-    CREATE_JOBS_COLLECTION: 'createJobsCollection',
-    CREATE_CITIES_COLLECTION: 'createCitiesCollection',
-    CREATE_AMOUNT_OF_JOBS_PER_DEPARTMENT_COLLECTION: 'createAmountOfJobsPerDepartmentCollection',
+    AGGREGATE_JOBS_BY_FIELD_TO_CMS: 'aggregateJobsByFieldToCMS',
+    REFERENCE_JOBS: 'referenceJobs',
+    CREATE_COLLECTIONS: 'createCollections',
     CREATE_API_KEY_COLLECTION_AND_FILL_IT: 'createApiKeyCollectionAndFillIt',
 }
 
@@ -23,39 +19,21 @@ const TASKS = {
     [TASKS_NAMES.SYNC_JOBS]: {
       name: TASKS_NAMES.SYNC_JOBS,
       childTasks: [
-        { name: TASKS_NAMES.CREATE_JOBS_COLLECTION },
-        { name: TASKS_NAMES.CREATE_CITIES_COLLECTION },
-        {name:  TASKS_NAMES.CREATE_AMOUNT_OF_JOBS_PER_DEPARTMENT_COLLECTION},
+        { name: TASKS_NAMES.CREATE_COLLECTIONS},
         { name: TASKS_NAMES.INSERT_JOBS_TO_CMS },
         { name: TASKS_NAMES.INSERT_JOBS_DESCRIPTIONS_LOCATION_APPLY_URL_TO_CMS },
-        { name: TASKS_NAMES.FILL_JOBS_PER_CITY_COLLECTION },
-        { name: TASKS_NAMES.FILL_JOBS_PER_DEPARTMENT_COLLECTION },
-        { name: TASKS_NAMES.REFERENCE_JOBS_TO_LOCATIONS },
-        { name: TASKS_NAMES.REFERENCE_JOBS_TO_DEPARTMENT },
+        { name: TASKS_NAMES.AGGREGATE_JOBS_BY_FIELD_TO_CMS },
+        {name: TASKS_NAMES.REFERENCE_JOBS},
       ],
       scheduleChildrenSequentially: true,
-      estimatedDurationSec: 30,
+      estimatedDurationSec: 60,
     },
-    [TASKS_NAMES.CREATE_JOBS_COLLECTION]: {
-      name: TASKS_NAMES.CREATE_JOBS_COLLECTION,
+    [TASKS_NAMES.CREATE_COLLECTIONS]: {
+      name: TASKS_NAMES.CREATE_COLLECTIONS,
       getIdentifier:()=> "SHOULD_NEVER_SKIP",
-      process:()=>createCollectionIfMissing(COLLECTIONS.JOBS, COLLECTIONS_FIELDS.JOBS,{ insert: 'ADMIN', update: 'ADMIN', remove: 'ADMIN', read: 'ANYONE' }),
+      process:createCollections,
       shouldSkipCheck:()=>false,
-      estimatedDurationSec:3
-    },
-    [TASKS_NAMES.CREATE_CITIES_COLLECTION]: {
-      name: TASKS_NAMES.CREATE_CITIES_COLLECTION,
-      getIdentifier:()=> "SHOULD_NEVER_SKIP",
-      process:()=>createCollectionIfMissing(COLLECTIONS.CITIES, COLLECTIONS_FIELDS.CITIES),
-      shouldSkipCheck:()=>false,
-      estimatedDurationSec:3
-    },
-    [TASKS_NAMES.CREATE_AMOUNT_OF_JOBS_PER_DEPARTMENT_COLLECTION]: {
-      name: TASKS_NAMES.CREATE_AMOUNT_OF_JOBS_PER_DEPARTMENT_COLLECTION,
-      getIdentifier:()=> "SHOULD_NEVER_SKIP",
-      process:()=>createCollectionIfMissing(COLLECTIONS.AMOUNT_OF_JOBS_PER_DEPARTMENT, COLLECTIONS_FIELDS.AMOUNT_OF_JOBS_PER_DEPARTMENT),
-      shouldSkipCheck:()=>false,
-      estimatedDurationSec:3
+      estimatedDurationSec:7
     },
     [TASKS_NAMES.INSERT_JOBS_TO_CMS]: {
       name: TASKS_NAMES.INSERT_JOBS_TO_CMS,
@@ -71,34 +49,20 @@ const TASKS = {
       shouldSkipCheck:()=>false,
       estimatedDurationSec:20
     },
-    [TASKS_NAMES.FILL_JOBS_PER_CITY_COLLECTION]: {
-        name: TASKS_NAMES.FILL_JOBS_PER_CITY_COLLECTION,
-        getIdentifier:()=> "SHOULD_NEVER_SKIP",
-        process:()=>aggregateJobsByFieldToCMS({ field: JOBS_COLLECTION_FIELDS.CITY_TEXT, collection: COLLECTIONS.CITIES }),
-        shouldSkipCheck:()=>false,
-        estimatedDurationSec:3
-      },
-      [TASKS_NAMES.FILL_JOBS_PER_DEPARTMENT_COLLECTION]: {
-        name: TASKS_NAMES.FILL_JOBS_PER_DEPARTMENT_COLLECTION,
-        getIdentifier:()=> "SHOULD_NEVER_SKIP",
-        process:()=>aggregateJobsByFieldToCMS({ field: JOBS_COLLECTION_FIELDS.DEPARTMENT, collection: COLLECTIONS.AMOUNT_OF_JOBS_PER_DEPARTMENT }),
-        shouldSkipCheck:()=>false,
-        estimatedDurationSec:3
-      },
-      [TASKS_NAMES.REFERENCE_JOBS_TO_LOCATIONS]: {
-        name: TASKS_NAMES.REFERENCE_JOBS_TO_LOCATIONS,
-        getIdentifier:()=> "SHOULD_NEVER_SKIP",
-        process:()=>referenceJobsToField({ referenceField: JOBS_COLLECTION_FIELDS.CITY, sourceCollection: COLLECTIONS.CITIES, jobField: JOBS_COLLECTION_FIELDS.CITY_TEXT }),
-        shouldSkipCheck:()=>false,
-        estimatedDurationSec:3
-      },
-      [TASKS_NAMES.REFERENCE_JOBS_TO_DEPARTMENT]: {
-        name: TASKS_NAMES.REFERENCE_JOBS_TO_DEPARTMENT,
-        getIdentifier:()=> "SHOULD_NEVER_SKIP",
-        process:()=>referenceJobsToField({ referenceField: JOBS_COLLECTION_FIELDS.DEPARTMENT_REF, sourceCollection: COLLECTIONS.AMOUNT_OF_JOBS_PER_DEPARTMENT, jobField: JOBS_COLLECTION_FIELDS.DEPARTMENT }),
-        shouldSkipCheck:()=>false,
-        estimatedDurationSec:3
-      },
+    [TASKS_NAMES.AGGREGATE_JOBS_BY_FIELD_TO_CMS]: {
+      name: TASKS_NAMES.AGGREGATE_JOBS_BY_FIELD_TO_CMS,
+      getIdentifier:()=> "SHOULD_NEVER_SKIP",
+      process:aggregateJobs,
+      shouldSkipCheck:()=>false,
+      estimatedDurationSec:6
+    },
+    [TASKS_NAMES.REFERENCE_JOBS]: {
+      name: TASKS_NAMES.REFERENCE_JOBS,
+      getIdentifier:()=> "SHOULD_NEVER_SKIP",
+      process:referenceJobs,
+      shouldSkipCheck:()=>false,
+      estimatedDurationSec:6
+    },
       [TASKS_NAMES.CREATE_API_KEY_COLLECTION_AND_FILL_IT]: {
         name: TASKS_NAMES.CREATE_API_KEY_COLLECTION_AND_FILL_IT,
         getIdentifier:()=> "SHOULD_NEVER_SKIP",

package/backend/data.js

@@ -18,7 +18,7 @@ function validatePosition(position) {
   if (!position.department || !position.department.label) {
     throw new Error('Position department is required and must have a label');
   }
-  if (!position.location || !position.location.city ||  typeof position.location.remote !== 'boolean') {
+  if (!position.location || !position.location.city || typeof position.location.remote !== 'boolean') {
     throw new Error('Position location is required and must have a city and remote');
   }
 
@@ -34,7 +34,7 @@ async function saveJobsDataToCMS() {
     const basicJob = {
       _id: position.id,
       title: position.title,
-      department: position.department.label.replace('&', ' and '),
+      department: position.department.label,
       cityText: normalizeCityName(position.location.city),
       location: position.location,
       country: position.location.country,
@@ -43,6 +43,7 @@ async function saveJobsDataToCMS() {
       postingStatus: position.postingStatus,
       jobDescription: null, // Will be filled later
     };
+    console.log("basicJob", basicJob);
     return basicJob;
   });
   console.log("jobsData@@@@@@@", jobsData);
@@ -133,6 +134,10 @@ async function saveJobsDescriptionsAndLocationApplyUrlToCMS() {
         console.log(
           `  API chunk ${chunkNumber} completed: ${chunkSuccesses} success, ${chunkFailures} failed`
         );
+        if (chunkNumber * API_CHUNK_SIZE < jobsWithNoDescriptions.items.length) {
+          console.log('  Waiting 2 seconds before next API chunk...');
+          await delay(2000);
+        }
       },
     });
 
@@ -157,7 +162,6 @@ async function saveJobsDescriptionsAndLocationApplyUrlToCMS() {
   }
 }
 
-
 function iterateOverAllJobs(results, field) {
   const jobsPerField = {};
   const cityLocations = {};
@@ -299,8 +303,36 @@ async function createApiKeyCollectionAndFillIt() {
     console.log("Smart token inserted into the ApiKey collection");
 }
 
+async function createCollections() {
+  console.log("Creating collections");
+  Promise.all(
+  [createCollectionIfMissing(COLLECTIONS.JOBS, JOBS_COLLECTION_FIELDS.JOBS,{ insert: 'ADMIN', update: 'ADMIN', remove: 'ADMIN', read: 'ANYONE' }),
+  createCollectionIfMissing(COLLECTIONS.CITIES, COLLECTIONS_FIELDS.CITIES),
+  createCollectionIfMissing(COLLECTIONS.AMOUNT_OF_JOBS_PER_DEPARTMENT, COLLECTIONS_FIELDS.AMOUNT_OF_JOBS_PER_DEPARTMENT)
+]);
+  console.log("finished creating Collections");
+}
+
+async function aggregateJobs() {
+  console.log("Aggregating jobs");
+  Promise.all([
+    aggregateJobsByFieldToCMS({ field: JOBS_COLLECTION_FIELDS.CITY_TEXT, collection: COLLECTIONS.CITIES }),
+    aggregateJobsByFieldToCMS({ field: JOBS_COLLECTION_FIELDS.DEPARTMENT, collection: COLLECTIONS.AMOUNT_OF_JOBS_PER_DEPARTMENT })
+  ]);
+  console.log("finished aggregating jobs");
+}
+
+async function referenceJobs() {
+  console.log("Reference jobs");
+  await referenceJobsToField({ referenceField: JOBS_COLLECTION_FIELDS.DEPARTMENT_REF, sourceCollection: COLLECTIONS.AMOUNT_OF_JOBS_PER_DEPARTMENT, jobField: JOBS_COLLECTION_FIELDS.DEPARTMENT })
+  await referenceJobsToField({ referenceField: JOBS_COLLECTION_FIELDS.CITY, sourceCollection: COLLECTIONS.CITIES, jobField: JOBS_COLLECTION_FIELDS.CITY_TEXT }),
+  console.log("finished referencing jobs");
+}
 
 module.exports = {
+  referenceJobs,
+  aggregateJobs,
+  createCollections,
     saveJobsDataToCMS,
     saveJobsDescriptionsAndLocationApplyUrlToCMS,
     aggregateJobsByFieldToCMS,

package/backend/fetchPositionsFromSRAPI.js

@@ -3,7 +3,7 @@ const { items: wixData } = require('@wix/data');
 const { COLLECTIONS } = require('./collectionConsts');
 async function makeSmartRecruitersRequest(path,token) {
    const baseUrl = 'https://api.smartrecruiters.com'; // PROD
-//  const baseUrl = 'https://bayank2.wixstudio.com/my-site-3//_functions'; // TEST
+  //const baseUrl = 'https://bayank2.wixstudio.com/my-site-3//_functions'; // TEST
   const fullUrl = `${baseUrl}${path}`;
   
     //console.log(`Making request to: ${fullUrl}`);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sr-npm",
-  "version": "1.7.118",
+  "version": "1.7.119",
   "description": "",
   "main": "index.js",
   "scripts": {