sr-npm 1.7.63 → 1.7.65

package/backend/collectionConsts.js

@@ -1,7 +1,7 @@
 const COLLECTIONS = {
-    AMOUNT_OF_JOBS_PER_DEPARTMENT: 'AmountOfJobsPerDepartment1',
-    CITIES: 'cities1',
-    JOBS: 'Jobs1',
+    AMOUNT_OF_JOBS_PER_DEPARTMENT: 'AmountOfJobsPerDepartment',
+    CITIES: 'cities',
+    JOBS: 'Jobs',
     API_KEY: 'ApiKey',
 }
 
@@ -9,31 +9,29 @@ const COLLECTIONS_FIELDS = {
     AMOUNT_OF_JOBS_PER_DEPARTMENT: [
       {key:'title', type: 'TEXT'},
       { key: 'count', type: 'NUMBER' },
+      { key: 'image', type: 'IMAGE' },
     ],
     CITIES: [
-      {key:'title', type: 'TEXT'},
-      { key: 'regionCode', type: 'TEXT' },
+      { key: 'title', type: 'TEXT' },
       { key: 'city', type: 'TEXT' },
-      {key:'location', type: 'OBJECT'},
+      {key:'locationAddress', type: 'ADDRESS'},
       {key:'count', type: 'NUMBER'},
       {key:'country', type: 'TEXT'},
-      {key:'remote', type: 'TEXT'},
-      {key:'countryCode', type: 'TEXT'},
-      {key:'manual', type: 'TEXT'},      
-      {key:'region', type: 'TEXT'},
-      {key:'latitude', type: 'NUMBER'},
-      {key:'longitude', type: 'NUMBER'},
     ],
     JOBS: [
         {key:'location', type: 'OBJECT'},
+        {key:'title', type: 'TEXT'},
+        {key:'locationAddress', type: 'ADDRESS'},
         {key:'postingStatus', type: 'TEXT'},
         {key:'country', type: 'TEXT'},
         {key:'department', type: 'TEXT'},
         {key:'language', type: 'TEXT'},
+        {key:'remote', type: 'BOOLEAN'},
         {key:'jobDescription', type: 'OBJECT'},  
-        {key:'cityText', type: 'TEXT'},         
-        {key:'departmentRef', type: 'REFERENCE', typeMetadata: { reference: { referencedCollectionId: 'AmountOfJobsPerDepartment1' } } },
-        {key:'city', type: 'REFERENCE', typeMetadata: { reference: { referencedCollectionId: 'cities1' } } },
+        {key:'cityText', type: 'TEXT'},
+        {key:'applyLink', type: 'URL'},         
+        {key:'departmentref', type: 'REFERENCE', typeMetadata: { reference: { referencedCollectionId: 'AmountOfJobsPerDepartment' } } },
+        {key:'city', type: 'REFERENCE', typeMetadata: { reference: { referencedCollectionId: 'cities' } } },
     ],  
     API_KEY: [
       {key:'token', type: 'TEXT'},

package/backend/consts.js

@@ -1,19 +1,10 @@
-const {
-  saveJobsDataToCMS,
-  saveJobsDescriptionsAndLocationToCMS,
-  aggregateJobsByFieldToCMS,
-  referenceJobsToField,
-  createApiKeyCollectionAndFillIt,
-} = require('./data');
+const {saveDataJobsToCMS,saveJobsDescriptionsAndLocationApplyUrlToCMS,aggregateJobsByFieldToCMS,referenceJobsToField,createApiKeyCollectionAndFillIt} = require('./data');
 const { createCollectionIfMissing } = require('@hisense-staging/velo-npm/backend');
 const { COLLECTIONS, COLLECTIONS_FIELDS } = require('./collectionConsts');
-
-const QUERY_MAX_LIMIT = 1000;
-
 const TASKS_NAMES = {
     SYNC_JOBS: 'syncJobsFromSRAPIToCMS',
     INSERT_JOBS_TO_CMS: 'insertJobsToCMS',
-    INSERT_JOBS_DESCRIPTIONS_TO_CMS: 'insertJobsDescriptionsToCMS',
+    INSERT_JOBS_DESCRIPTIONS_LOCATION_APPLY_URL_TO_CMS: 'insertJobsDescriptionsLocationApplyUrlToCMS',
     FILL_JOBS_PER_CITY_COLLECTION: 'fillJobsPerCityCollection',
     FILL_JOBS_PER_DEPARTMENT_COLLECTION: 'fillJobsPerDepartmentCollection',
     REFERENCE_JOBS_TO_LOCATIONS: 'referenceJobsToLocations',
@@ -26,126 +17,103 @@ const TASKS_NAMES = {
 
 
 const TASKS = {
-  [TASKS_NAMES.SYNC_JOBS]: {
-    name: TASKS_NAMES.SYNC_JOBS,
-    childTasks: [
-      { name: TASKS_NAMES.CREATE_JOBS_COLLECTION },
-      { name: TASKS_NAMES.CREATE_CITIES_COLLECTION },
-      { name: TASKS_NAMES.CREATE_AMOUNT_OF_JOBS_PER_DEPARTMENT_COLLECTION },
-      { name: TASKS_NAMES.INSERT_JOBS_TO_CMS },
-      { name: TASKS_NAMES.INSERT_JOBS_DESCRIPTIONS_TO_CMS },
-      { name: TASKS_NAMES.FILL_JOBS_PER_CITY_COLLECTION },
-      { name: TASKS_NAMES.FILL_JOBS_PER_DEPARTMENT_COLLECTION },
-      { name: TASKS_NAMES.REFERENCE_JOBS_TO_LOCATIONS },
-      { name: TASKS_NAMES.REFERENCE_JOBS_TO_DEPARTMENT },
-    ],
-    scheduleChildrenSequentially: true,
-    estimatedDurationSec: 30,
-  },
-  [TASKS_NAMES.CREATE_JOBS_COLLECTION]: {
-    name: TASKS_NAMES.CREATE_JOBS_COLLECTION,
-    getIdentifier: () => 'SHOULD_NEVER_SKIP',
-    process: () => createCollectionIfMissing(COLLECTIONS.JOBS, COLLECTIONS_FIELDS.JOBS),
-    shouldSkipCheck: () => false,
-    estimatedDurationSec: 3,
-  },
-  [TASKS_NAMES.CREATE_CITIES_COLLECTION]: {
-    name: TASKS_NAMES.CREATE_CITIES_COLLECTION,
-    getIdentifier: () => 'SHOULD_NEVER_SKIP',
-    process: () => createCollectionIfMissing(COLLECTIONS.CITIES, COLLECTIONS_FIELDS.CITIES),
-    shouldSkipCheck: () => false,
-    estimatedDurationSec: 3,
-  },
-  [TASKS_NAMES.CREATE_AMOUNT_OF_JOBS_PER_DEPARTMENT_COLLECTION]: {
-    name: TASKS_NAMES.CREATE_AMOUNT_OF_JOBS_PER_DEPARTMENT_COLLECTION,
-    getIdentifier: () => 'SHOULD_NEVER_SKIP',
-    process: () =>
-      createCollectionIfMissing(
-        COLLECTIONS.AMOUNT_OF_JOBS_PER_DEPARTMENT,
-        COLLECTIONS_FIELDS.AMOUNT_OF_JOBS_PER_DEPARTMENT
-      ),
-    shouldSkipCheck: () => false,
-    estimatedDurationSec: 3,
-  },
-  [TASKS_NAMES.INSERT_JOBS_TO_CMS]: {
-    name: TASKS_NAMES.INSERT_JOBS_TO_CMS,
-    getIdentifier: () => 'SHOULD_NEVER_SKIP',
-    process: saveJobsDataToCMS,
-    shouldSkipCheck: () => false,
-    estimatedDurationSec: 20,
-  },
-  [TASKS_NAMES.INSERT_JOBS_DESCRIPTIONS_TO_CMS]: {
-    name: TASKS_NAMES.INSERT_JOBS_DESCRIPTIONS_TO_CMS,
-    getIdentifier: () => 'SHOULD_NEVER_SKIP',
-    process: saveJobsDescriptionsAndLocationToCMS,
-    shouldSkipCheck: () => false,
-    estimatedDurationSec: 20,
-  },
-  [TASKS_NAMES.FILL_JOBS_PER_CITY_COLLECTION]: {
-    name: TASKS_NAMES.FILL_JOBS_PER_CITY_COLLECTION,
-    getIdentifier: () => 'SHOULD_NEVER_SKIP',
-    process: () =>
-      aggregateJobsByFieldToCMS({
-        field: COLLECTIONS_FIELDS.JOBS[6].key,
-        collection: COLLECTIONS.CITIES,
-      }),
-    shouldSkipCheck: () => false,
-    estimatedDurationSec: 3,
-  },
-  [TASKS_NAMES.FILL_JOBS_PER_DEPARTMENT_COLLECTION]: {
-    name: TASKS_NAMES.FILL_JOBS_PER_DEPARTMENT_COLLECTION,
-    getIdentifier: () => 'SHOULD_NEVER_SKIP',
-    process: () =>
-      aggregateJobsByFieldToCMS({
-        field: COLLECTIONS_FIELDS.JOBS[3].key,
-        collection: COLLECTIONS.AMOUNT_OF_JOBS_PER_DEPARTMENT,
-      }),
-    shouldSkipCheck: () => false,
-    estimatedDurationSec: 3,
-  },
-  [TASKS_NAMES.REFERENCE_JOBS_TO_LOCATIONS]: {
-    name: TASKS_NAMES.REFERENCE_JOBS_TO_LOCATIONS,
-    getIdentifier: () => 'SHOULD_NEVER_SKIP',
-    process: () =>
-      referenceJobsToField({
-        referenceField: COLLECTIONS_FIELDS.JOBS[8].key,
-        sourceCollection: COLLECTIONS.CITIES,
-        jobField: COLLECTIONS_FIELDS.JOBS[6].key,
-      }),
-    shouldSkipCheck: () => false,
-    estimatedDurationSec: 3,
-  },
-  [TASKS_NAMES.REFERENCE_JOBS_TO_DEPARTMENT]: {
-    name: TASKS_NAMES.REFERENCE_JOBS_TO_DEPARTMENT,
-    getIdentifier: () => 'SHOULD_NEVER_SKIP',
-    process: () =>
-      referenceJobsToField({
-        referenceField: COLLECTIONS_FIELDS.JOBS[7].key,
-        sourceCollection: COLLECTIONS.AMOUNT_OF_JOBS_PER_DEPARTMENT,
-        jobField: COLLECTIONS_FIELDS.JOBS[3].key,
-      }),
-    shouldSkipCheck: () => false,
-    estimatedDurationSec: 3,
-  },
-  [TASKS_NAMES.CREATE_API_KEY_COLLECTION_AND_FILL_IT]: {
-    name: TASKS_NAMES.CREATE_API_KEY_COLLECTION_AND_FILL_IT,
-    getIdentifier:()=> "SHOULD_NEVER_SKIP",
-    process:()=>createApiKeyCollectionAndFillIt(),
-    shouldSkipCheck:()=>false,
-    estimatedDurationSec:3
-  }
-};
+    [TASKS_NAMES.SYNC_JOBS]: {
+      name: TASKS_NAMES.SYNC_JOBS,
+      childTasks: [
+        { name: TASKS_NAMES.CREATE_JOBS_COLLECTION },
+        { name: TASKS_NAMES.CREATE_CITIES_COLLECTION },
+        {name:  TASKS_NAMES.CREATE_AMOUNT_OF_JOBS_PER_DEPARTMENT_COLLECTION},
+        { name: TASKS_NAMES.INSERT_JOBS_TO_CMS },
+        { name: TASKS_NAMES.INSERT_JOBS_DESCRIPTIONS_LOCATION_APPLY_URL_TO_CMS },
+        { name: TASKS_NAMES.FILL_JOBS_PER_CITY_COLLECTION },
+        { name: TASKS_NAMES.FILL_JOBS_PER_DEPARTMENT_COLLECTION },
+        { name: TASKS_NAMES.REFERENCE_JOBS_TO_LOCATIONS },
+        { name: TASKS_NAMES.REFERENCE_JOBS_TO_DEPARTMENT },
+      ],
+      scheduleChildrenSequentially: true,
+      estimatedDurationSec: 30,
+    },
+    [TASKS_NAMES.CREATE_JOBS_COLLECTION]: {
+      name: TASKS_NAMES.CREATE_JOBS_COLLECTION,
+      getIdentifier:()=> "SHOULD_NEVER_SKIP",
+      process:()=>createCollectionIfMissing(COLLECTIONS.JOBS, COLLECTIONS_FIELDS.JOBS,{ insert: 'ADMIN', update: 'ADMIN', remove: 'ADMIN', read: 'ANYONE' }),
+      shouldSkipCheck:()=>false,
+      estimatedDurationSec:3
+    },
+    [TASKS_NAMES.CREATE_CITIES_COLLECTION]: {
+      name: TASKS_NAMES.CREATE_CITIES_COLLECTION,
+      getIdentifier:()=> "SHOULD_NEVER_SKIP",
+      process:()=>createCollectionIfMissing(COLLECTIONS.CITIES, COLLECTIONS_FIELDS.CITIES),
+      shouldSkipCheck:()=>false,
+      estimatedDurationSec:3
+    },
+    [TASKS_NAMES.CREATE_AMOUNT_OF_JOBS_PER_DEPARTMENT_COLLECTION]: {
+      name: TASKS_NAMES.CREATE_AMOUNT_OF_JOBS_PER_DEPARTMENT_COLLECTION,
+      getIdentifier:()=> "SHOULD_NEVER_SKIP",
+      process:()=>createCollectionIfMissing(COLLECTIONS.AMOUNT_OF_JOBS_PER_DEPARTMENT, COLLECTIONS_FIELDS.AMOUNT_OF_JOBS_PER_DEPARTMENT),
+      shouldSkipCheck:()=>false,
+      estimatedDurationSec:3
+    },
+    [TASKS_NAMES.INSERT_JOBS_TO_CMS]: {
+      name: TASKS_NAMES.INSERT_JOBS_TO_CMS,
+      getIdentifier:()=> "SHOULD_NEVER_SKIP",
+      process:saveDataJobsToCMS,
+      shouldSkipCheck:()=>false,
+      estimatedDurationSec:20
+    },
+    [TASKS_NAMES.INSERT_JOBS_DESCRIPTIONS_LOCATION_APPLY_URL_TO_CMS]: {
+      name: TASKS_NAMES.INSERT_JOBS_DESCRIPTIONS_LOCATION_APPLY_URL_TO_CMS,
+      getIdentifier:()=> "SHOULD_NEVER_SKIP",
+      process:saveJobsDescriptionsAndLocationApplyUrlToCMS,
+      shouldSkipCheck:()=>false,
+      estimatedDurationSec:20
+    },
+    [TASKS_NAMES.FILL_JOBS_PER_CITY_COLLECTION]: {
+        name: TASKS_NAMES.FILL_JOBS_PER_CITY_COLLECTION,
+        getIdentifier:()=> "SHOULD_NEVER_SKIP",
+        process:()=>aggregateJobsByFieldToCMS({ field: 'cityText', collection: COLLECTIONS.CITIES }),
+        shouldSkipCheck:()=>false,
+        estimatedDurationSec:3
+      },
+      [TASKS_NAMES.FILL_JOBS_PER_DEPARTMENT_COLLECTION]: {
+        name: TASKS_NAMES.FILL_JOBS_PER_DEPARTMENT_COLLECTION,
+        getIdentifier:()=> "SHOULD_NEVER_SKIP",
+        process:()=>aggregateJobsByFieldToCMS({ field: 'department', collection: COLLECTIONS.AMOUNT_OF_JOBS_PER_DEPARTMENT }),
+        shouldSkipCheck:()=>false,
+        estimatedDurationSec:3
+      },
+      [TASKS_NAMES.REFERENCE_JOBS_TO_LOCATIONS]: {
+        name: TASKS_NAMES.REFERENCE_JOBS_TO_LOCATIONS,
+        getIdentifier:()=> "SHOULD_NEVER_SKIP",
+        process:()=>referenceJobsToField({ referenceField: 'city', sourceCollection: COLLECTIONS.CITIES, jobField: 'cityText' }),
+        shouldSkipCheck:()=>false,
+        estimatedDurationSec:3
+      },
+      [TASKS_NAMES.REFERENCE_JOBS_TO_DEPARTMENT]: {
+        name: TASKS_NAMES.REFERENCE_JOBS_TO_DEPARTMENT,
+        getIdentifier:()=> "SHOULD_NEVER_SKIP",
+        process:()=>referenceJobsToField({ referenceField: 'departmentref', sourceCollection: COLLECTIONS.AMOUNT_OF_JOBS_PER_DEPARTMENT, jobField: 'department' }),
+        shouldSkipCheck:()=>false,
+        estimatedDurationSec:3
+      },
+      [TASKS_NAMES.CREATE_API_KEY_COLLECTION_AND_FILL_IT]: {
+        name: TASKS_NAMES.CREATE_API_KEY_COLLECTION_AND_FILL_IT,
+        getIdentifier:()=> "SHOULD_NEVER_SKIP",
+        process:()=>createApiKeyCollectionAndFillIt(),
+        shouldSkipCheck:()=>false,
+        estimatedDurationSec:3
+      }
+}
 
 
 
 const TASK_TYPE = {
-  SCHEDULED: 'scheduled',
-  EVENT: 'event',
-};
+    SCHEDULED: 'scheduled',
+    EVENT: 'event',
+  };
 
   module.exports = {
     TASKS_NAMES,
     TASK_TYPE,
     TASKS,
-    QUERY_MAX_LIMIT
 };

package/backend/data.js

@@ -1,257 +1,322 @@
 const { items: wixData } = require('@wix/data');
 const { fetchPositionsFromSRAPI, fetchJobDescription } = require('./fetchPositionsFromSRAPI');
+const { chunkedBulkOperation } = require('./utils');
 const { createCollectionIfMissing } = require('@hisense-staging/velo-npm/backend');
 const { COLLECTIONS, COLLECTIONS_FIELDS } = require('./collectionConsts');
 const { secrets } = require("@wix/secrets");
 const { auth } = require('@wix/essentials');
-const { chunkedBulkOperation, delay, countJobsPerGivenField, fillCityLocation ,prepateToSaveArray,normalizeCityName} = require('./utils');
-const { getAllPositions } = require('./queries');
 
-
-
-async function saveJobsDataToCMS() {
-  const positions = await fetchPositionsFromSRAPI();
-  // bulk insert to jobs collection without descriptions first
-  const jobsData = positions.content.map(position => {
-    const basicJob = {
-      _id: position.id,
-      title: position.title,
-      department: position.department.label,
-      cityText: normalizeCityName(position.location.city),
-      location: position.location,
-      country: position.location.country,
-      remote: position.location.remote,
-      language: position.language.label,
-      postingStatus: position.postingStatus,
-      jobDescription: null, // Will be filled later
-    };
-    return basicJob;
-  });
-
-  const chunkSize = 1000;
-  let totalSaved = 0;
-  const totalChunks = Math.ceil(jobsData.length / chunkSize);
-
-  console.log(
-    `Processing ${jobsData.length} jobs in ${totalChunks} chunks of max ${chunkSize} items each`
-  );
-
-  await chunkedBulkOperation({
-    items: jobsData,
-    chunkSize,
-    processChunk: async (chunk, chunkNumber) => {
-      console.log(`Saving chunk ${chunkNumber}/${totalChunks}: ${chunk.length} jobs`);
-      try {
-        const result = await wixData.bulkSave('Jobs1', chunk);
-        const saved = result.inserted + result.updated || chunk.length;
-        totalSaved += saved;
-        console.log(
-          `✓ Chunk ${chunkNumber} saved successfully. Inserted: ${result.inserted}, Updated: ${result.updated}`
-        );
-      } catch (error) {
-        console.error(`✗ Error saving chunk ${chunkNumber}:`, error);
-        throw error;
-      }
-    },
-  });
-
-  console.log(`✓ All chunks processed. Total jobs saved: ${totalSaved}/${jobsData.length}`);
+// Utility function to normalize city names
+function normalizeCityName(city) {
+    if (!city) return city;
+    // Remove accents/diacritics, trim whitespace
+    return city.normalize('NFD').replace(/\p{Diacritic}/gu, '').trim();
 }
 
-async function saveJobsDescriptionsAndLocationToCMS() {
-  console.log('🚀 Starting job descriptions update process for ALL jobs');
-
-  try {
-    let jobsWithNoDescriptions = await getJobsWithNoDescriptions();
-    let totalUpdated = 0;
-    let totalFailed = 0;
-    let totalProcessed = 0;
-
-    console.log(
-      `Total jobs in database without descriptions:  ${jobsWithNoDescriptions?.items?.length}`
-    );
-
-    if (jobsWithNoDescriptions.items.length === 0) {
-      console.log('No jobs found in database');
-      return { success: true, message: 'No jobs found' };
-    }
-
-
-    const API_CHUNK_SIZE = 80;
-    const pageChunks = Math.ceil(jobsWithNoDescriptions.items.length / API_CHUNK_SIZE);
+async function saveDataJobsToCMS() {
+    const positions = await fetchPositionsFromSRAPI();
+    // bulk insert to jobs collection without descriptions first
+    const jobsData = positions.content.map((position) => {
+        const basicJob = {
+            _id: position.id,
+            title: position.title,
+            department: position.department.label,
+            cityText: normalizeCityName(position.location.city),
+            location: position.location,
+            country: position.location.country,
+            remote: position.location.remote,
+            language: position.language.label,
+            postingStatus: position.postingStatus,
+            jobDescription: null // Will be filled later
+        }
+        return basicJob;
+    });
+    
+    const chunkSize = 1000;
+    let totalSaved = 0;
+    const totalChunks = Math.ceil(jobsData.length / chunkSize);
+    
+    console.log(`Processing ${jobsData.length} jobs in ${totalChunks} chunks of max ${chunkSize} items each`);
 
     await chunkedBulkOperation({
-      items: jobsWithNoDescriptions.items,
-      chunkSize: API_CHUNK_SIZE,
-      processChunk: async (chunk, chunkNumber) => {
-        console.log(`  Processing API chunk ${chunkNumber}/${pageChunks} (${chunk.length} jobs)`);
-        const chunkPromises = chunk.map(async job => {
-          try {
-            const jobDetails = await fetchJobDescription(job._id);
-            const jobLocation = fetchJobLocation(jobDetails);
-
-            const updatedJob = {
-              ...job,
-              locationAddress: jobLocation,
-              jobDescription: jobDetails.jobAd.sections,
-            };
-            await wixData.update('Jobs1', updatedJob);
-            return { success: true, jobId: job._id, title: job.title };
-          } catch (error) {
-            console.error(`    ❌ Failed to update ${job.title} (${job._id}):`, error);
-            return { success: false, jobId: job._id, title: job.title, error: error.message };
-          }
-        });
-        const chunkResults = await Promise.all(chunkPromises);
-        const chunkSuccesses = chunkResults.filter(r => r.success).length;
-        const chunkFailures = chunkResults.filter(r => !r.success).length;
-        totalUpdated += chunkSuccesses;
-        totalFailed += chunkFailures;
-        totalProcessed += chunk.length;
-        console.log(
-          `  API chunk ${chunkNumber} completed: ${chunkSuccesses} success, ${chunkFailures} failed`
-        );
-        if (chunkNumber * API_CHUNK_SIZE < jobsWithNoDescriptions.items.length) {
-          console.log('  Waiting 2 seconds before next API chunk...');
-          await delay(2000);
+        items: jobsData,
+        chunkSize,
+        processChunk: async (chunk, chunkNumber) => {
+            console.log(`Saving chunk ${chunkNumber}/${totalChunks}: ${chunk.length} jobs`);
+            try {
+                const result = await wixData.bulkSave(COLLECTIONS.JOBS, chunk);
+                const saved = result.inserted + result.updated || chunk.length;
+                totalSaved += saved;
+                console.log(`✓ Chunk ${chunkNumber} saved successfully. Inserted: ${result.inserted}, Updated: ${result.updated}`);
+            } catch (error) {
+                console.error(`✗ Error saving chunk ${chunkNumber}:`, error);
+                throw error;
+            }
         }
-      },
     });
+    
+    console.log(`✓ All chunks processed. Total jobs saved: ${totalSaved}/${jobsData.length}`);
 
-
-
-    console.log(`\n✅ Finished updating ALL job descriptions`);
-    console.log(`📊 Final Results:`);
-    console.log(`   Total jobs processed: ${totalProcessed}`);
-    console.log(`   Total updated: ${totalUpdated}`);
-    console.log(`   Total failed: ${totalFailed}`);
-
-    return {
-      success: true,
-      totalProcessed: totalProcessed,
-      totalUpdated: totalUpdated,
-      totalFailed: totalFailed,
-      message: `Successfully updated ${totalUpdated} job descriptions out of ${totalProcessed} total jobs`,
-    };
-  } catch (error) {
-    console.error('❌ Error in updateJobDescriptions:', error);
-    throw error;
-  }
 }
 
+async function saveJobsDescriptionsAndLocationApplyUrlToCMS() {
+    
+   console.log('🚀 Starting job descriptions update process for ALL jobs using pagination...');
+    
+    try {
+        let jobsWithNoDescriptions = await getJobsWithNoDescriptions();
+        let totalUpdated = 0;
+        let totalFailed = 0;
+        let totalProcessed = 0;
+        let pageNumber = 1;
+        
+        // Start with the first page query - limit to 100 jobs per page
+       // let jobsQuery = await wixData.query("Jobs").limit(300).find();
+        
+        console.log(`Total jobs in database without descriptions:  ${jobsWithNoDescriptions.totalCount}`);
+        
+        if (jobsWithNoDescriptions.totalCount === 0) {
+            console.log('No jobs found in database');
+            return { success: true, message: 'No jobs found' };
+        }
 
-function iterateOverAllJobs(results, field) {
-  const jobsPerField = {};
-  const cityLocations = {};
-    countJobsPerGivenField(results, field, jobsPerField);
-    if (field === 'cityText') {
-      fillCityLocation(results, cityLocations);
+        // Process all pages using hasNext() pagination
+        do {
+            const currentPageJobs = jobsWithNoDescriptions.items;
+            console.log(`\n📄 Processing page ${pageNumber} with ${currentPageJobs.length} jobs...`);
+            
+            // Process jobs in smaller chunks of 5 for API calls within each page
+            const API_CHUNK_SIZE = 80;
+            const pageChunks = Math.ceil(currentPageJobs.length / API_CHUNK_SIZE);
+            
+            await chunkedBulkOperation({
+                items: currentPageJobs,
+                chunkSize: API_CHUNK_SIZE,
+                processChunk: async (chunk, chunkNumber) => {
+                    console.log(`  Processing API chunk ${chunkNumber}/${pageChunks} (${chunk.length} jobs)`);
+                    const chunkPromises = chunk.map(async (job) => {
+                        try {
+                            //   console.log(`    Fetching description for: ${job.title} (${job._id})`);
+                            const jobDetails = await fetchJobDescription(job._id);
+                            const jobLocation = fetchJobLocation(jobDetails)
+                            const applyLink = fetchApplyLink(jobDetails);
+                            
+                            const updatedJob = {
+                                ...job,
+                                locationAddress: jobLocation,
+                                jobDescription: jobDetails.jobAd.sections,
+                                applyLink: applyLink
+                            };
+                            await wixData.update(COLLECTIONS.JOBS, updatedJob);
+                            // console.log(`    ✅ Updated description for: ${job.title}`);
+                            return { success: true, jobId: job._id, title: job.title };
+                        } catch (error) {
+                            console.error(`    ❌ Failed to update ${job.title} (${job._id}):`, error);
+                            return { success: false, jobId: job._id, title: job.title, error: error.message };
+                        }
+                    });
+                    const chunkResults = await Promise.all(chunkPromises);
+                    const chunkSuccesses = chunkResults.filter(r => r.success).length;
+                    const chunkFailures = chunkResults.filter(r => !r.success).length;
+                    totalUpdated += chunkSuccesses;
+                    totalFailed += chunkFailures;
+                    totalProcessed += chunk.length;
+                    console.log(`  API chunk ${chunkNumber} completed: ${chunkSuccesses} success, ${chunkFailures} failed`);
+                    if (chunkNumber * API_CHUNK_SIZE < currentPageJobs.length) {
+                        console.log('  Waiting 2 seconds before next API chunk...');
+                        await new Promise(resolve => setTimeout(resolve, 2000));
+                    }
+                }
+            });
+            
+            console.log(`📄 Page ${pageNumber} completed. Updated: ${totalUpdated}, Failed: ${totalFailed}, Total processed: ${totalProcessed}/${jobsWithNoDescriptions.totalCount}`);
+            
+            // Check if there are more pages and get the next page
+            if (jobsWithNoDescriptions.hasNext()) {
+                console.log('🔄 Moving to next page...');
+                jobsWithNoDescriptions = await jobsWithNoDescriptions.next();
+                pageNumber++;
+                
+                // Add a delay between pages
+                console.log('Waiting 3 seconds before next page...');
+                await new Promise(resolve => setTimeout(resolve, 3000));
+            }
+            
+        } while (jobsWithNoDescriptions.hasNext());
+        
+        console.log(`\n✅ Finished updating ALL job descriptions using pagination!`);
+        console.log(`📊 Final Results:`);
+        console.log(`   Total pages processed: ${pageNumber}`);
+        console.log(`   Total jobs processed: ${totalProcessed}`);
+        console.log(`   Total updated: ${totalUpdated}`);
+        console.log(`   Total failed: ${totalFailed}`);
+        
+        return {
+            success: true,
+            totalPages: pageNumber,
+            totalProcessed: totalProcessed,
+            totalUpdated: totalUpdated,
+            totalFailed: totalFailed,
+            message: `Successfully updated ${totalUpdated} job descriptions out of ${totalProcessed} total jobs across ${pageNumber} pages`
+        };
+        
+    } catch (error) {
+        console.error('❌ Error in updateJobDescriptions:', error);
+        throw error;
     }
-    return { jobsPerField, cityLocations };
 }
 
 async function aggregateJobsByFieldToCMS({ field, collection }) {
-  console.log(`counting jobs per ${field}.`);
-  let results = await getAllPositions();
-  const { jobsPerField, cityLocations } = iterateOverAllJobs(results, field);
-  const toSave = prepateToSaveArray(jobsPerField, cityLocations, field);
-  if (toSave.length === 0) {
-    console.log('No jobs found.');
-    return { success: true, message: 'No jobs to save.' };
-  }
-  try {
-    const saveResult = await wixData.bulkSave(collection, toSave);
-    console.log(`Saved ${toSave.length} ${field} counts to ${collection}.`);
-    return { success: true, saved: toSave.length, result: saveResult };
-  } catch (err) {
-    console.error(`Error saving jobs per ${field}:`, err);
-    return { success: false, error: err.message };
-  }
+    console.log(`counting jobs per ${field}.`);
+    let jobsPerField = {};
+    let cityLocations = {};
+    let query = wixData.query(COLLECTIONS.JOBS).limit(1000);
+    let results = await query.find();
+    const cityLocationAddress={}
+    let page = 1;
+    do {
+        console.log(`Page ${page}: ${results.items.length} jobs.`);
+        for (const job of results.items) {
+            if (!job[field])
+                {
+                    throw new Error(`Job ${job._id} has no ${field} field`);
+                } 
+            jobsPerField[job[field]] = (jobsPerField[job[field]] || 0) + 1;
+            if (field === 'cityText' && !cityLocationAddress[job[field]] && !cityLocations[job[field]]) {
+                cityLocationAddress[job[field]] = job.locationAddress;
+                cityLocations[job[field]] = job.location;
+            }
+        }
+        if (results.hasNext()) {
+            results = await results.next();
+            page++;
+        }
+    } while (results.hasNext());
+    let toSave = [];
+    if (field === 'cityText') {
+         toSave = Object.entries(jobsPerField).map(([value, amount]) => {
+            const locAddress = cityLocationAddress[value] || {};
+            const loc = cityLocations[value] || {};
+            value = normalizeCityName(value);
+
+            return {
+                title: value,
+                _id: value.replace(/\s+/g, ''),
+                count: amount,
+                locationAddress: locAddress,
+                country: loc.country,
+                city: loc.city,
+            };
+        });
+    }
+    else{
+    // Prepare array for bulkSave
+     toSave = Object.entries(jobsPerField).map(([value, amount]) => ({
+        title: value,
+        _id: value.replace(/\s+/g, ''),
+        count: amount
+    }));
+    }
+    if (toSave.length === 0) {
+        console.log('No jobs found.');
+        return { success: true, message: 'No jobs to save.' };
+    }
+
+    try {
+        const saveResult = await wixData.bulkSave(collection, toSave);
+        console.log(`Saved ${toSave.length} ${field} counts to ${collection}.`);
+        return { success: true, saved: toSave.length, result: saveResult };
+    } catch (err) {
+        console.error(`Error saving jobs per ${field}:`, err);
+        return { success: false, error: err.message };
+    }
 }
 
 async function getJobsWithNoDescriptions() {
-  let jobswithoutdescriptionsQuery = await wixData
-    .query('Jobs1')
-    .limit(1000)
-    .isEmpty('jobDescription')
-    .find();
-  return jobswithoutdescriptionsQuery;
+    
+   let jobswithoutdescriptionsQuery = await wixData.query(COLLECTIONS.JOBS).limit(1000).isEmpty("jobDescription").find(); // with 900 as the limit, 429 error won't happen
+    return jobswithoutdescriptionsQuery;
 }
 
-/**
- * @param {Object} params
- * @param {"city"|"departmentRef"} params.referenceField
- * @param {"cities1"|"AmountOfJobsPerDepartment1"} params.sourceCollection
- * @param {"cityText"|"department"} params.jobField
- */
-async function referenceJobsToField({ referenceField, sourceCollection, jobField }) {
-  // Fetch all source items (cities or departments)
-  const sources = await wixData.query(sourceCollection).limit(1000).find();
-  const sourceMap = {};
-  for (const item of sources.items) {
-    sourceMap[item.title] = item._id;
-  }
-
-  // Fetch all jobs
-  let jobsResults = await getAllPositions();
-  let jobsToUpdate = [];
-  
-  for (const job of jobsResults) {
-    const refId = sourceMap[job[jobField]];
-    if (refId) {
-      jobsToUpdate.push({
-        ...job,
-        [referenceField]: refId,
-      });
+async function referenceJobsToField({
+    referenceField,      // e.g., "city" or "department"
+    sourceCollection,    // e.g., "cities" or "departments"
+    jobField,            // e.g., "cityText" or "department"
+    
+}) {
+    // Fetch all source items (cities or departments)
+    const sources = await wixData.query(sourceCollection).limit(1000).find();
+    const sourceMap = {};
+    for (const item of sources.items) {
+        sourceMap[item.title] = item._id;
     }
-  }
 
-  // Remove system fields that cannot be updated
-  jobsToUpdate = jobsToUpdate.map(job => {
-    const { _createdDate, _updatedDate, ...rest } = job;
-    return rest;
-  });
+    // Fetch all jobs
+    let jobsResults = await wixData.query(COLLECTIONS.JOBS).limit(1000).find();
+    let jobsToUpdate = [];
+    console.log('jobsResults',jobsResults.items);
+
+    do {
+        for (const job of jobsResults.items) {
+            const refId = sourceMap[job[jobField]];
+            if (refId) {
+                jobsToUpdate.push({
+                    ...job,
+                    [referenceField]: refId
+                });
+            }
+        }
+        if (jobsResults.hasNext()) {
+            jobsResults = await jobsResults.next();
+        } else {
+            break;
+        }
+    } while (true);
+
+    // Remove system fields that cannot be updated
+    jobsToUpdate = jobsToUpdate.map(job => {
+        const { _createdDate, _updatedDate, ...rest } = job;
+        return rest;
+    });
+
+    // Bulk update in chunks of 1000
+    const chunkSize = 1000;
+    await chunkedBulkOperation({
+        items: jobsToUpdate,
+        chunkSize,
+        processChunk: async (chunk) => {
+            await wixData.bulkUpdate(COLLECTIONS.JOBS, chunk);
+        }
+    });
 
-  // Bulk update in chunks of 1000
-  const chunkSize = 1000;
-  await chunkedBulkOperation({
-    items: jobsToUpdate,
-    chunkSize,
-    processChunk: async chunk => {
-      await wixData.bulkUpdate('Jobs1', chunk);
-    },
-  });
+    return { success: true, updated: jobsToUpdate.length };
+}
 
-  return { success: true, updated: jobsToUpdate.length };
+function fetchApplyLink(jobDetails) {
+    return jobDetails.actions.applyOnWeb.url;
 }
 
 function fetchJobLocation(jobDetails) {
-  const isZeroLocation =
-    jobDetails.location.latitude === '0.0000' && jobDetails.location.longitude === '0.0000';
-  const jobLocation = {
-    location: isZeroLocation
-      ? {}
-      : {
-          latitude: parseFloat(jobDetails.location.latitude),
-          longitude: parseFloat(jobDetails.location.longitude),
+    const jobLocation = {
+        location:  {
+            latitude: parseFloat(jobDetails.location.latitude),
+            longitude: parseFloat(jobDetails.location.longitude)
         },
-    city: jobDetails.location.city,
-    country: jobDetails.location.country,
-    formatted: [
-      jobDetails.location.city,
-      jobDetails.location.region,
-      jobDetails.location.regionCode,
-      jobDetails.location.country,
-    ]
-      .filter(Boolean)
-      .join(', '),
-    streetAddress: {},
-    subdivision: '',
-    postalCode: '',
-  };
-  return jobLocation;
-}
+        city: jobDetails.location.city,
+        country: jobDetails.location.country,
+        formatted: [
+            jobDetails.location.city,
+            jobDetails.location.region,
+            jobDetails.location.regionCode,
+            jobDetails.location.country
+        ].filter(Boolean).join(', '),
+        streetAddress: {},
+        subdivision: "",
+        postalCode: ""
+    };
+    return jobLocation;
 
+}
 
 
 function getSmartToken() {
@@ -268,7 +333,7 @@ function getSmartToken() {
 
 async function createApiKeyCollectionAndFillIt() {
     console.log("Creating ApiKey collection and filling it with the smart token");
-    await createCollectionIfMissing(COLLECTIONS.API_KEY, COLLECTIONS_FIELDS.API_KEY,null,'singleItem');
+    await createCollectionIfMissing(COLLECTIONS.API_KEY, COLLECTIONS_FIELDS.API_KEY);
     console.log("Getting the smart token");
     const token = await getSmartToken();
     console.log("token is :  ", token);
@@ -282,9 +347,9 @@ async function createApiKeyCollectionAndFillIt() {
 
 
 module.exports = {
-    saveJobsDataToCMS,
-    saveJobsDescriptionsAndLocationToCMS,
+    saveDataJobsToCMS,
+    saveJobsDescriptionsAndLocationApplyUrlToCMS,
     aggregateJobsByFieldToCMS,
     referenceJobsToField,
     createApiKeyCollectionAndFillIt,
-};
+};

package/backend/fetchPositionsFromSRAPI.js

@@ -1,7 +1,6 @@
 const { fetch } = require('wix-fetch');
 const { items: wixData } = require('@wix/data');
 const { COLLECTIONS } = require('./collectionConsts');
-
 async function makeSmartRecruitersRequest(path,token) {
   // const baseUrl = 'https://api.smartrecruiters.com'; // PROD
   const baseUrl = 'https://aoxley54.wixstudio.com/external-template/_functions'; // TEST
@@ -35,7 +34,7 @@ async function fetchPositionsFromSRAPI() {
   let allPositions = [];
   let totalFound = 0;
   let nextPageId = null; // Start with no page ID for the first request
-  let page = 0;
+  let pageCount = 0;
   const MAX_PAGES = 30 // Safety limit to prevent infinite loops
   const token = await getSmartTokenFromCMS();
 
@@ -43,44 +42,45 @@ async function fetchPositionsFromSRAPI() {
 
   do {
     try {
-      page++;
-
+      pageCount++;
+      
       // Build the API path - first request has no page parameter, subsequent use nextPageId
       let apiPath = '/jobs?limit=50';
       if (nextPageId) {
         apiPath += `&nextPageId=${nextPageId}`;
       }
       
-      console.log(`Fetching page ${page} with path: ${apiPath}`);
+      console.log(`Fetching page ${pageCount} with path: ${apiPath}`);
       const response = await makeSmartRecruitersRequest(apiPath,token);
       
       // Add positions from this page to our collection
       if (response.content && Array.isArray(response.content)) {
         allPositions = allPositions.concat(response.content);
-        console.log(`Page ${page}: Found ${response.content.length} positions`);
+        console.log(`Page ${pageCount}: Found ${response.content.length} positions`);
       }
-
+      
       // Update total count from first response
-      if (page === 1) {
+      if (pageCount === 1) {
         totalFound = response.totalFound || 0;
         console.log(`Total positions available: ${totalFound}`);
       }
-
+      
       // Get the nextPageId for the next iteration
-      nextPageId = response.nextPageId && response.nextPageId !== '' ? response.nextPageId : null;
-
+      nextPageId = response.nextPageId && response.nextPageId !== "" ? response.nextPageId : null;
+      
       if (nextPageId) {
         console.log(`Next page ID: ${nextPageId}`);
       } else {
         console.log('No more pages to fetch');
       }
+      
     } catch (error) {
-      console.error(`Error fetching page ${page}:`, error);
+      console.error(`Error fetching page ${pageCount}:`, error);
       throw error;
     }
-
+    
     // Safety check to prevent infinite loops
-    if (page >= MAX_PAGES) {
+    if (pageCount >= MAX_PAGES) {
       console.warn(`Reached maximum page limit of ${MAX_PAGES}. Stopping pagination.`);
       break;
     }
@@ -89,12 +89,12 @@ async function fetchPositionsFromSRAPI() {
   console.log(`Finished fetching all pages. Total positions collected: ${allPositions.length}`);
 
   // Return response in the same format as before, but with all positions
-  const result = {
+  const result =  {
     totalFound: totalFound,
     offset: 0,
     limit: allPositions.length,
-    nextPageId: '', // Always empty since we've fetched everything
-    content: allPositions,
+    nextPageId: "", // Always empty since we've fetched everything
+    content: allPositions
   };
 
   const amountOfUniqueJobs = new Set(allPositions.map(job => job.id)).size;
@@ -108,7 +108,8 @@ async function fetchPositionsFromSRAPI() {
 }
 
 async function fetchJobDescription(jobId) {
-  return await makeSmartRecruitersRequest(`/jobs/${jobId}`);
+  const token = await getSmartTokenFromCMS();
+  return await makeSmartRecruitersRequest(`/jobs/${jobId}`,token);
 }
 
 async function getSmartTokenFromCMS() {
@@ -122,6 +123,6 @@ async function getSmartTokenFromCMS() {
 
 
 module.exports = {
-  fetchPositionsFromSRAPI,
-  fetchJobDescription,
+    fetchPositionsFromSRAPI,
+    fetchJobDescription,
 };

package/backend/index.js

@@ -1,6 +1,7 @@
 module.exports = {
-  ...require('./utils'),
-  ...require('./queries'),
-  ...require('./fetchPositionsFromSRAPI'),
-  ...require('./consts'),
-};
+    ...require('./utils'),
+    ...require('./queries'),
+    ...require('./fetchPositionsFromSRAPI'),
+    ...require('./consts'),
+  };
+  

package/backend/queries.js

@@ -1,22 +1,19 @@
-const { items: wixData } = require('@wix/data');
+const { COLLECTIONS } = require('./collectionConsts');
+const { items } = require('@wix/data');
+
 
 async function getAllPositions() {
-  return wixData
-    .query('Jobs1')
-    .limit(1000)
-    .find()
-    .then(result => result.items);
+    return await items.query(COLLECTIONS.JOBS).find().items;
+    
 }
 
 async function getPositionsByField(field, value) {
-  return wixData
-    .query('Jobs1')
-    .eq(field, value)
-    .find()
-    .then(result => result.items);
+    return await items.query(COLLECTIONS.JOBS).where(field, value).find().items;
+    
 }
 
+
 module.exports = {
-  getAllPositions,
-  getPositionsByField,
-};
+    getAllPositions,
+    getPositionsByField,
+};

package/backend/utils.js

@@ -1,75 +1,11 @@
 async function chunkedBulkOperation({ items, chunkSize, processChunk }) {
-  for (let i = 0; i < items.length; i += chunkSize) {
-    const chunk = items.slice(i, i + chunkSize);
-    await processChunk(chunk, Math.floor(i / chunkSize) + 1);
-  }
-}
-
-async function delay(ms) {
-  await new Promise(resolve => setTimeout(resolve, ms));
-}
-
-function countJobsPerGivenField(jobs, field,jobsPerField) {
-  for (const job of jobs) {
-    if (!job[field]) {
-      throw new Error(`Job ${job._id} has no ${field} field`);
-    }
-    jobsPerField[job[field]] = (jobsPerField[job[field]] || 0) + 1;
-  }
-}
-
-function fillCityLocation(jobs, cityLocations) {
-  for (const job of jobs) {
-    if (!cityLocations[job.cityText]) {
-      cityLocations[job.cityText] = job.location;
+    for (let i = 0; i < items.length; i += chunkSize) {
+        const chunk = items.slice(i, i + chunkSize);
+        await processChunk(chunk, Math.floor(i / chunkSize) + 1);
     }
-  }
 }
 
-function prepareToSaveArray(jobsPerField, cityLocations, field) {
-  if (field === 'cityText') {
-    return Object.entries(jobsPerField).map(([value, amount]) => {
-      const loc = cityLocations[value] || {};
-      value = normalizeCityName(value);
-      return {
-        title: value,
-        _id: value.replace(/\s+/g, ''),
-        count: amount,
-        location: loc,
-        countryCode: loc.countryCode,
-        country: loc.country,
-        region: loc.region,
-        city: loc.city,
-        manual: loc.manual.toString(),
-        remote: loc.remote.toString(),
-        regionCode: loc.regionCode,
-        latitude: loc.latitude,
-        longitude: loc.longitude,
-      };
-    });
-  } else {
-    return Object.entries(jobsPerField).map(([value, amount]) => ({
-      title: value,
-      _id: value.replace(/\s+/g, ''),
-      count: amount,
-    }));
-  }
-}
-
-function normalizeCityName(city) {
-  if (!city) return city;
-  // Remove accents/diacritics, trim whitespace
-  return city
-    .normalize('NFD')
-    .replace(/\p{Diacritic}/gu, '')
-    .trim();
-}
 
 module.exports = {
-  chunkedBulkOperation,
-  delay,
-  countJobsPerGivenField,
-  fillCityLocation,
-  prepateToSaveArray: prepareToSaveArray,
-  normalizeCityName,
-};
+    chunkedBulkOperation,
+};

package/index.js

@@ -1,5 +1,6 @@
 module.exports = {
-  ...require('./public'),
-  ...require('./pages'),
-  ...require('./backend'),
-};
+    ...require('./public'),
+    ...require('./pages'),
+    ...require('./backend'),
+  };
+  

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sr-npm",
-  "version": "1.7.63",
+  "version": "1.7.65",
   "description": "",
   "main": "index.js",
   "scripts": {
@@ -17,12 +17,9 @@
   },
   "homepage": "https://github.com/psdevteamenterprise/sr-npm#readme",
   "dependencies": {
-    "@hisense-staging/velo-npm": "3.0.19",
+    "@hisense-staging/velo-npm": "1.7.251",
     "@wix/data": "^1.0.211",
     "@wix/essentials": "^0.1.24",
-    "@wix/secrets": "^1.0.53"
-  },
-  "devDependencies": {
-    "prettier": "^3.6.2"
+    "@wix/secrets": "1.0.53"
   }
 }

package/.prettierrc

@@ -1,16 +0,0 @@
-{
-  "singleQuote": true,
-  "trailingComma": "es5",
-  "printWidth": 100,
-  "tabWidth": 2,
-  "semi": true,
-  "bracketSpacing": true,
-  "arrowParens": "avoid",
-  "endOfLine": "lf",
-  "quoteProps": "as-needed",
-  "jsxSingleQuote": false,
-  "jsxBracketSameLine": false,
-  "requirePragma": false,
-  "insertPragma": false,
-  "proseWrap": "preserve"
-}