sr-npm 1.7.1313 → 1.7.1314

package/backend/data.js

@@ -176,34 +176,34 @@ async function saveJobsDataToCMS() {
     return titleA.localeCompare(titleB, undefined, { sensitivity: 'base', numeric: true });
   });
 
-  const chunkSize = 1000;
-  let totalSaved = 0;
-  const totalChunks = Math.ceil(jobsData.length / chunkSize);
+  // const chunkSize = 1000;
+  // let totalSaved = 0;
+  // const totalChunks = Math.ceil(jobsData.length / chunkSize);
 
-  console.log(
-    `Processing ${jobsData.length} jobs in ${totalChunks} chunks of max ${chunkSize} items each`
-  );
-  console.log("truncating jobs collection");
-  await wixData.truncate(COLLECTIONS.JOBS);
-  await chunkedBulkOperation({
-    items: jobsData,
-    chunkSize,
-    processChunk: async (chunk, chunkNumber) => {
-      console.log(`Saving chunk ${chunkNumber}/${totalChunks}: ${chunk.length} jobs`);
-      try {
-        const result = await wixData.bulkSave(COLLECTIONS.JOBS, chunk);
-        const saved = result.inserted + result.updated || chunk.length;
-        totalSaved += saved;
-        console.log(
-          `✓ Chunk ${chunkNumber} saved successfully. Inserted: ${result.inserted}, Updated: ${result.updated}`
-        );
-      } catch (error) {
-        console.error(`✗ Error saving chunk ${chunkNumber}:`, error);
-        throw error;
-      }
-    },
-  });
-  console.log(`✓ All chunks processed. Total jobs saved: ${totalSaved}/${jobsData.length}`);
+  // console.log(
+  //   `Processing ${jobsData.length} jobs in ${totalChunks} chunks of max ${chunkSize} items each`
+  // );
+  
+  // await chunkedBulkOperation({
+  //   items: jobsData,
+  //   chunkSize,
+  //   processChunk: async (chunk, chunkNumber) => {
+  //     console.log(`Saving chunk ${chunkNumber}/${totalChunks}: ${chunk.length} jobs`);
+  //     try {
+  //       const result = await wixData.bulkSave(COLLECTIONS.JOBS, chunk);
+  //       const saved = result.inserted + result.updated || chunk.length;
+  //       totalSaved += saved;
+  //       console.log(
+  //         `✓ Chunk ${chunkNumber} saved successfully. Inserted: ${result.inserted}, Updated: ${result.updated}`
+  //       );
+  //     } catch (error) {
+  //       console.error(`✗ Error saving chunk ${chunkNumber}:`, error);
+  //       throw error;
+  //     }
+  //   },
+  // });
+  // console.log(`✓ All chunks processed. Total jobs saved: ${totalSaved}/${jobsData.length}`);
+  return jobsData;
 }
 
 async function insertJobsReference(id, valueId) {
@@ -262,9 +262,9 @@ async function saveJobsDescriptionsAndLocationApplyUrlReferencesToCMS() {
       console.log("inserted jobs references to custom values collection successfully");
     }
   
-    let totalUpdated = 0;
-    let totalFailed = 0;
-    let totalProcessed = 0;
+    // let totalUpdated = 0;
+    // let totalFailed = 0;
+    // let totalProcessed = 0;
 
     console.log(
       `Total jobs in database without descriptions:  ${jobsWithNoDescriptions?.items?.length}`
@@ -276,17 +276,12 @@ async function saveJobsDescriptionsAndLocationApplyUrlReferencesToCMS() {
     }
 
 
-    const API_CHUNK_SIZE = 80;
-    const pageChunks = Math.ceil(jobsWithNoDescriptions.items.length / API_CHUNK_SIZE);
+   // const API_CHUNK_SIZE = 80;
+    //const pageChunks = Math.ceil(jobsWithNoDescriptions.items.length / API_CHUNK_SIZE);
     const richContentConverterToken = await getTokenFromCMS(TOKEN_NAME.RICH_CONTENT_CONVERTER_TOKEN);
-    await chunkedBulkOperation({
-      items: jobsWithNoDescriptions.items,
-      chunkSize: API_CHUNK_SIZE,
-      processChunk: async (chunk, chunkNumber) => {
-        console.log(`  Processing API chunk ${chunkNumber}/${pageChunks} (${chunk.length} jobs)`);
-        const chunkPromises = chunk.map(async job => {
-          try {
-            const jobDetails = await fetchJobDescription(job._id);
+
+    jobsWithNoDescriptions=jobsWithNoDescriptions.map(async job => {
+      const jobDetails = await fetchJobDescription(job._id);
             const richContentDescription=await htmlRichContentConverter(jobDetails.jobAd.sections,richContentConverterToken);
             const jobLocation = fetchJobLocation(jobDetails);
             const {applyLink , referFriendLink} = fetchApplyAndReferFriendLink(jobDetails);
@@ -297,40 +292,62 @@ async function saveJobsDescriptionsAndLocationApplyUrlReferencesToCMS() {
               applyLink: applyLink,
               referFriendLink: referFriendLink,
             };
-            await wixData.update(COLLECTIONS.JOBS, updatedJob);
-            return { success: true, jobId: job._id, title: job.title };
-          } catch (error) {
-            console.error(`    ❌ Failed to update ${job.title} (${job._id}):`, error);
-            return { success: false, jobId: job._id, title: job.title, error: error.message };
-          }
-        });
-        const chunkResults = await Promise.all(chunkPromises);
-        const chunkSuccesses = chunkResults.filter(r => r.success).length;
-        const chunkFailures = chunkResults.filter(r => !r.success).length;
-        totalUpdated += chunkSuccesses;
-        totalFailed += chunkFailures;
-        totalProcessed += chunk.length;
-        console.log(
-          `  API chunk ${chunkNumber} completed: ${chunkSuccesses} success, ${chunkFailures} failed`
-        );
-      },
-    });
-
-
-
-    console.log(`\n✅ Finished updating ALL job descriptions`);
-    console.log(`📊 Final Results:`);
-    console.log(`   Total jobs processed: ${totalProcessed}`);
-    console.log(`   Total updated: ${totalUpdated}`);
-    console.log(`   Total failed: ${totalFailed}`);
-
-    return {
-      success: true,
-      totalProcessed: totalProcessed,
-      totalUpdated: totalUpdated,
-      totalFailed: totalFailed,
-      message: `Successfully updated ${totalUpdated} job descriptions out of ${totalProcessed} total jobs`,
-    };
+            return updatedJob;
+    })
+    return {success: true, updatedJobs: jobsWithNoDescriptions};
+
+    // await chunkedBulkOperation({
+    //   items: jobsWithNoDescriptions.items,
+    //   chunkSize: API_CHUNK_SIZE,
+    //   processChunk: async (chunk, chunkNumber) => {
+    //     console.log(`  Processing API chunk ${chunkNumber}/${pageChunks} (${chunk.length} jobs)`);
+    //     const chunkPromises = chunk.map(async job => {
+    //       try {
+    //         const jobDetails = await fetchJobDescription(job._id);
+    //         const richContentDescription=await htmlRichContentConverter(jobDetails.jobAd.sections,richContentConverterToken);
+    //         const jobLocation = fetchJobLocation(jobDetails);
+    //         const {applyLink , referFriendLink} = fetchApplyAndReferFriendLink(jobDetails);
+    //         const updatedJob = {
+    //           ...job,
+    //           locationAddress: jobLocation,
+    //           jobDescription: richContentDescription,
+    //           applyLink: applyLink,
+    //           referFriendLink: referFriendLink,
+    //         };
+    //         await wixData.update(COLLECTIONS.JOBS, updatedJob);
+    //         return { success: true, jobId: job._id, title: job.title };
+    //       } catch (error) {
+    //         console.error(`    ❌ Failed to update ${job.title} (${job._id}):`, error);
+    //         return { success: false, jobId: job._id, title: job.title, error: error.message };
+    //       }
+    //     });
+    //     const chunkResults = await Promise.all(chunkPromises);
+    //     const chunkSuccesses = chunkResults.filter(r => r.success).length;
+    //     const chunkFailures = chunkResults.filter(r => !r.success).length;
+    //     totalUpdated += chunkSuccesses;
+    //     totalFailed += chunkFailures;
+    //     totalProcessed += chunk.length;
+    //     console.log(
+    //       `  API chunk ${chunkNumber} completed: ${chunkSuccesses} success, ${chunkFailures} failed`
+    //     );
+    //   },
+    // });
+
+
+
+    // console.log(`\n✅ Finished updating ALL job descriptions`);
+    // console.log(`📊 Final Results:`);
+    // console.log(`   Total jobs processed: ${totalProcessed}`);
+    // console.log(`   Total updated: ${totalUpdated}`);
+    // console.log(`   Total failed: ${totalFailed}`);
+
+    // return {
+    //   success: true,
+    //   totalProcessed: totalProcessed,
+    //   totalUpdated: totalUpdated,
+    //   totalFailed: totalFailed,
+    //   message: `Successfully updated ${totalUpdated} job descriptions out of ${totalProcessed} total jobs`,
+    // };
   } catch (error) {
     console.error('❌ Error in updateJobDescriptions:', error);
     throw error;
@@ -507,20 +524,56 @@ async function syncJobsFast() {
   try{
   console.log("Syncing jobs fast");
   await createCollections();
-  await clearCollections();
+  // await clearCollections();
   await fillSecretManagerMirror();
 
-  console.log("saving jobs data to CMS");
-  await saveJobsDataToCMS();
-  console.log("saved jobs data to CMS successfully");
+  console.log("saving jobs data to CMS");//change comment
+  const jobsData = await saveJobsDataToCMS(); //change name
+  console.log("jobsData inside syncJobsFast: ",jobsData);
+  console.log("saved jobs data to CMS successfully");//change comment
+  
+  console.log("saving jobs descriptions and location apply url to CMS");//change comment
+  const status=await saveJobsDescriptionsAndLocationApplyUrlReferencesToCMS();
+
+  console.log("status inside syncJobsFast: ",status);
+  if(status.success){
+    await clearCollections();
+    const updatedJobs=status.updatedJobs;
+    const chunkSize = 1000;
+  let totalSaved = 0;
+  const totalChunks = Math.ceil(updatedJobs.length / chunkSize);
+
+  console.log(
+    `Processing ${updatedJobs.length} jobs in ${totalChunks} chunks of max ${chunkSize} items each`
+  );
   
-  console.log("saving jobs descriptions and location apply url to CMS");
-  await saveJobsDescriptionsAndLocationApplyUrlReferencesToCMS();
+  await chunkedBulkOperation({
+    items: updatedJobs,
+    chunkSize,
+    processChunk: async (chunk, chunkNumber) => {
+      console.log(`Saving chunk ${chunkNumber}/${totalChunks}: ${chunk.length} jobs`);
+      try {
+        const result = await wixData.bulkSave(COLLECTIONS.JOBS, chunk);
+        const saved = result.inserted + result.updated || chunk.length;
+        totalSaved += saved;
+        console.log(
+          `✓ Chunk ${chunkNumber} saved successfully. Inserted: ${result.inserted}, Updated: ${result.updated}`
+        );
+      } catch (error) {
+        console.error(`✗ Error saving chunk ${chunkNumber}:`, error);
+        throw error;
+      }
+    },
+  });
+  console.log(`✓ All chunks processed. Total jobs saved: ${totalSaved}/${updatedJobs.length}`);
+  }
+  console.log("saved jobs descriptions and location apply url to CMS successfully");//change comment
 
-  console.log("saved jobs descriptions and location apply url to CMS successfully");
   await aggregateJobs();
   await referenceJobs();
+
   console.log("syncing jobs fast finished successfully");
+
   }
   catch (error) {
     error.message="Error syncing jobs: "+error.message;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sr-npm",
-  "version": "1.7.1313",
+  "version": "1.7.1314",
   "description": "",
   "main": "index.js",
   "scripts": {