sr-npm 1.7.34 → 1.7.36

package/.prettierrc

@@ -0,0 +1,16 @@
+{
+  "singleQuote": true,
+  "trailingComma": "es5",
+  "printWidth": 100,
+  "tabWidth": 2,
+  "semi": true,
+  "bracketSpacing": true,
+  "arrowParens": "avoid",
+  "endOfLine": "lf",
+  "quoteProps": "as-needed",
+  "jsxSingleQuote": false,
+  "jsxBracketSameLine": false,
+  "requirePragma": false,
+  "insertPragma": false,
+  "proseWrap": "preserve"
+}

package/backend/consts.js

@@ -1,119 +1,185 @@
-const {saveDataJobsToCMS,saveJobsDescriptionsAndLocationToCMS,aggregateJobsByFieldToCMS,referenceJobsToField,createApiKeyCollectionAndFillIt} = require('./data');
+const {
+  saveJobsDataToCMS,
+  saveJobsDescriptionsAndLocationToCMS,
+  aggregateJobsByFieldToCMS,
+  referenceJobsToField,
+} = require('./data');
 const { createCollectionIfMissing } = require('@hisense-staging/velo-npm/backend');
-const { COLLECTIONS, COLLECTIONS_FIELDS } = require('./collectionConsts');
-const TASKS_NAMES = {
-    SYNC_JOBS: 'syncJobsFromSRAPIToCMS',
-    INSERT_JOBS_TO_CMS: 'insertJobsToCMS',
-    INSERT_JOBS_DESCRIPTIONS_TO_CMS: 'insertJobsDescriptionsToCMS',
-    FILL_JOBS_PER_CITY_COLLECTION: 'fillJobsPerCityCollection',
-    FILL_JOBS_PER_DEPARTMENT_COLLECTION: 'fillJobsPerDepartmentCollection',
-    REFERENCE_JOBS_TO_LOCATIONS: 'referenceJobsToLocations',
-    REFERENCE_JOBS_TO_DEPARTMENT: 'referenceJobsToDepartment',
-    CREATE_JOBS_COLLECTION: 'createJobsCollection',
-    CREATE_CITIES_COLLECTION: 'createCitiesCollection',
-    CREATE_AMOUNT_OF_JOBS_PER_DEPARTMENT_COLLECTION: 'createAmountOfJobsPerDepartmentCollection',
-    CREATE_API_KEY_COLLECTION_AND_FILL_IT: 'createApiKeyCollectionAndFillIt',
-}
 
+const QUERY_MAX_LIMIT = 1000;
+
+const TASKS_NAMES = {
+  SYNC_JOBS: 'syncJobsFromSRAPIToCMS',
+  INSERT_JOBS_TO_CMS: 'insertJobsToCMS',
+  INSERT_JOBS_DESCRIPTIONS_TO_CMS: 'insertJobsDescriptionsToCMS',
+  FILL_JOBS_PER_CITY_COLLECTION: 'fillJobsPerCityCollection',
+  FILL_JOBS_PER_DEPARTMENT_COLLECTION: 'fillJobsPerDepartmentCollection',
+  REFERENCE_JOBS_TO_LOCATIONS: 'referenceJobsToLocations',
+  REFERENCE_JOBS_TO_DEPARTMENT: 'referenceJobsToDepartment',
+  CREATE_JOBS_COLLECTION: 'createJobsCollection',
+  CREATE_CITIES_COLLECTION: 'createCitiesCollection',
+  CREATE_AMOUNT_OF_JOBS_PER_DEPARTMENT_COLLECTION: 'createAmountOfJobsPerDepartmentCollection',
+};
 
 const TASKS = {
-    [TASKS_NAMES.SYNC_JOBS]: {
-      name: TASKS_NAMES.SYNC_JOBS,
-      childTasks: [
-        { name: TASKS_NAMES.CREATE_JOBS_COLLECTION },
-        { name: TASKS_NAMES.CREATE_CITIES_COLLECTION },
-        {name:  TASKS_NAMES.CREATE_AMOUNT_OF_JOBS_PER_DEPARTMENT_COLLECTION},
-        { name: TASKS_NAMES.INSERT_JOBS_TO_CMS },
-        { name: TASKS_NAMES.INSERT_JOBS_DESCRIPTIONS_TO_CMS },
-        { name: TASKS_NAMES.FILL_JOBS_PER_CITY_COLLECTION },
-        { name: TASKS_NAMES.FILL_JOBS_PER_DEPARTMENT_COLLECTION },
-        { name: TASKS_NAMES.REFERENCE_JOBS_TO_LOCATIONS },
-        { name: TASKS_NAMES.REFERENCE_JOBS_TO_DEPARTMENT },
-      ],
-      scheduleChildrenSequentially: true,
-      estimatedDurationSec: 30,
-    },
-    [TASKS_NAMES.CREATE_JOBS_COLLECTION]: {
-      name: TASKS_NAMES.CREATE_JOBS_COLLECTION,
-      getIdentifier:()=> "SHOULD_NEVER_SKIP",
-      process:()=>createCollectionIfMissing(COLLECTIONS.JOBS, COLLECTIONS_FIELDS.JOBS),
-      shouldSkipCheck:()=>false,
-      estimatedDurationSec:3
-    },
-    [TASKS_NAMES.CREATE_CITIES_COLLECTION]: {
-      name: TASKS_NAMES.CREATE_CITIES_COLLECTION,
-      getIdentifier:()=> "SHOULD_NEVER_SKIP",
-      process:()=>createCollectionIfMissing(COLLECTIONS.CITIES, COLLECTIONS_FIELDS.CITIES),
-      shouldSkipCheck:()=>false,
-      estimatedDurationSec:3
-    },
-    [TASKS_NAMES.CREATE_AMOUNT_OF_JOBS_PER_DEPARTMENT_COLLECTION]: {
-      name: TASKS_NAMES.CREATE_AMOUNT_OF_JOBS_PER_DEPARTMENT_COLLECTION,
-      getIdentifier:()=> "SHOULD_NEVER_SKIP",
-      process:()=>createCollectionIfMissing(COLLECTIONS.AMOUNT_OF_JOBS_PER_DEPARTMENT, COLLECTIONS_FIELDS.AMOUNT_OF_JOBS_PER_DEPARTMENT),
-      shouldSkipCheck:()=>false,
-      estimatedDurationSec:3
-    },
-    [TASKS_NAMES.INSERT_JOBS_TO_CMS]: {
-      name: TASKS_NAMES.INSERT_JOBS_TO_CMS,
-      getIdentifier:()=> "SHOULD_NEVER_SKIP",
-      process:saveDataJobsToCMS,
-      shouldSkipCheck:()=>false,
-      estimatedDurationSec:20
-    },
-    [TASKS_NAMES.INSERT_JOBS_DESCRIPTIONS_TO_CMS]: {
-      name: TASKS_NAMES.INSERT_JOBS_DESCRIPTIONS_TO_CMS,
-      getIdentifier:()=> "SHOULD_NEVER_SKIP",
-      process:saveJobsDescriptionsAndLocationToCMS,
-      shouldSkipCheck:()=>false,
-      estimatedDurationSec:20
-    },
-    [TASKS_NAMES.FILL_JOBS_PER_CITY_COLLECTION]: {
-        name: TASKS_NAMES.FILL_JOBS_PER_CITY_COLLECTION,
-        getIdentifier:()=> "SHOULD_NEVER_SKIP",
-        process:()=>aggregateJobsByFieldToCMS({ field: 'cityText', collection: 'cities' }),
-        shouldSkipCheck:()=>false,
-        estimatedDurationSec:3
-      },
-      [TASKS_NAMES.FILL_JOBS_PER_DEPARTMENT_COLLECTION]: {
-        name: TASKS_NAMES.FILL_JOBS_PER_DEPARTMENT_COLLECTION,
-        getIdentifier:()=> "SHOULD_NEVER_SKIP",
-        process:()=>aggregateJobsByFieldToCMS({ field: 'department', collection: 'AmountOfJobsPerDepartment' }),
-        shouldSkipCheck:()=>false,
-        estimatedDurationSec:3
-      },
-      [TASKS_NAMES.REFERENCE_JOBS_TO_LOCATIONS]: {
-        name: TASKS_NAMES.REFERENCE_JOBS_TO_LOCATIONS,
-        getIdentifier:()=> "SHOULD_NEVER_SKIP",
-        process:()=>referenceJobsToField({ referenceField: 'city', sourceCollection: 'cities', jobField: 'cityText' }),
-        shouldSkipCheck:()=>false,
-        estimatedDurationSec:3
-      },
-      [TASKS_NAMES.REFERENCE_JOBS_TO_DEPARTMENT]: {
-        name: TASKS_NAMES.REFERENCE_JOBS_TO_DEPARTMENT,
-        getIdentifier:()=> "SHOULD_NEVER_SKIP",
-        process:()=>referenceJobsToField({ referenceField: 'departmentref', sourceCollection: 'AmountOfJobsPerDepartment', jobField: 'department' }),
-        shouldSkipCheck:()=>false,
-        estimatedDurationSec:3
-      },
-      [TASKS_NAMES.CREATE_API_KEY_COLLECTION_AND_FILL_IT]: {
-        name: TASKS_NAMES.CREATE_API_KEY_COLLECTION_AND_FILL_IT,
-        getIdentifier:()=> "SHOULD_NEVER_SKIP",
-        process:()=>createApiKeyCollectionAndFillIt(),
-        shouldSkipCheck:()=>false,
-        estimatedDurationSec:3
-      }
-}
+  [TASKS_NAMES.SYNC_JOBS]: {
+    name: TASKS_NAMES.SYNC_JOBS,
+    childTasks: [
+      { name: TASKS_NAMES.CREATE_JOBS_COLLECTION },
+      { name: TASKS_NAMES.CREATE_CITIES_COLLECTION },
+      { name: TASKS_NAMES.CREATE_AMOUNT_OF_JOBS_PER_DEPARTMENT_COLLECTION },
+      { name: TASKS_NAMES.INSERT_JOBS_TO_CMS },
+      { name: TASKS_NAMES.INSERT_JOBS_DESCRIPTIONS_TO_CMS },
+      { name: TASKS_NAMES.FILL_JOBS_PER_CITY_COLLECTION },
+      { name: TASKS_NAMES.FILL_JOBS_PER_DEPARTMENT_COLLECTION },
+      { name: TASKS_NAMES.REFERENCE_JOBS_TO_LOCATIONS },
+      { name: TASKS_NAMES.REFERENCE_JOBS_TO_DEPARTMENT },
+    ],
+    scheduleChildrenSequentially: true,
+    estimatedDurationSec: 30,
+  },
+  [TASKS_NAMES.CREATE_JOBS_COLLECTION]: {
+    name: TASKS_NAMES.CREATE_JOBS_COLLECTION,
+    getIdentifier: () => 'SHOULD_NEVER_SKIP',
+    process: () => createCollectionIfMissing(COLLECTIONS.JOBS, COLLECTIONS_FIELDS.JOBS),
+    shouldSkipCheck: () => false,
+    estimatedDurationSec: 3,
+  },
+  [TASKS_NAMES.CREATE_CITIES_COLLECTION]: {
+    name: TASKS_NAMES.CREATE_CITIES_COLLECTION,
+    getIdentifier: () => 'SHOULD_NEVER_SKIP',
+    process: () => createCollectionIfMissing(COLLECTIONS.CITIES, COLLECTIONS_FIELDS.CITIES),
+    shouldSkipCheck: () => false,
+    estimatedDurationSec: 3,
+  },
+  [TASKS_NAMES.CREATE_AMOUNT_OF_JOBS_PER_DEPARTMENT_COLLECTION]: {
+    name: TASKS_NAMES.CREATE_AMOUNT_OF_JOBS_PER_DEPARTMENT_COLLECTION,
+    getIdentifier: () => 'SHOULD_NEVER_SKIP',
+    process: () =>
+      createCollectionIfMissing(
+        COLLECTIONS.AMOUNT_OF_JOBS_PER_DEPARTMENT,
+        COLLECTIONS_FIELDS.AMOUNT_OF_JOBS_PER_DEPARTMENT
+      ),
+    shouldSkipCheck: () => false,
+    estimatedDurationSec: 3,
+  },
+  [TASKS_NAMES.INSERT_JOBS_TO_CMS]: {
+    name: TASKS_NAMES.INSERT_JOBS_TO_CMS,
+    getIdentifier: () => 'SHOULD_NEVER_SKIP',
+    process: saveJobsDataToCMS,
+    shouldSkipCheck: () => false,
+    estimatedDurationSec: 20,
+  },
+  [TASKS_NAMES.INSERT_JOBS_DESCRIPTIONS_TO_CMS]: {
+    name: TASKS_NAMES.INSERT_JOBS_DESCRIPTIONS_TO_CMS,
+    getIdentifier: () => 'SHOULD_NEVER_SKIP',
+    process: saveJobsDescriptionsAndLocationToCMS,
+    shouldSkipCheck: () => false,
+    estimatedDurationSec: 20,
+  },
+  [TASKS_NAMES.FILL_JOBS_PER_CITY_COLLECTION]: {
+    name: TASKS_NAMES.FILL_JOBS_PER_CITY_COLLECTION,
+    getIdentifier: () => 'SHOULD_NEVER_SKIP',
+    process: () =>
+      aggregateJobsByFieldToCMS({
+        field: COLLECTIONS_FIELDS.JOBS[6].key,
+        collection: COLLECTIONS.CITIES,
+      }),
+    shouldSkipCheck: () => false,
+    estimatedDurationSec: 3,
+  },
+  [TASKS_NAMES.FILL_JOBS_PER_DEPARTMENT_COLLECTION]: {
+    name: TASKS_NAMES.FILL_JOBS_PER_DEPARTMENT_COLLECTION,
+    getIdentifier: () => 'SHOULD_NEVER_SKIP',
+    process: () =>
+      aggregateJobsByFieldToCMS({
+        field: COLLECTIONS_FIELDS.JOBS[3].key,
+        collection: COLLECTIONS.AMOUNT_OF_JOBS_PER_DEPARTMENT,
+      }),
+    shouldSkipCheck: () => false,
+    estimatedDurationSec: 3,
+  },
+  [TASKS_NAMES.REFERENCE_JOBS_TO_LOCATIONS]: {
+    name: TASKS_NAMES.REFERENCE_JOBS_TO_LOCATIONS,
+    getIdentifier: () => 'SHOULD_NEVER_SKIP',
+    process: () =>
+      referenceJobsToField({
+        referenceField: COLLECTIONS_FIELDS.JOBS[8].key,
+        sourceCollection: COLLECTIONS.CITIES,
+        jobField: COLLECTIONS_FIELDS.JOBS[6].key,
+      }),
+    shouldSkipCheck: () => false,
+    estimatedDurationSec: 3,
+  },
+  [TASKS_NAMES.REFERENCE_JOBS_TO_DEPARTMENT]: {
+    name: TASKS_NAMES.REFERENCE_JOBS_TO_DEPARTMENT,
+    getIdentifier: () => 'SHOULD_NEVER_SKIP',
+    process: () =>
+      referenceJobsToField({
+        referenceField: COLLECTIONS_FIELDS.JOBS[7].key,
+        sourceCollection: COLLECTIONS.AMOUNT_OF_JOBS_PER_DEPARTMENT,
+        jobField: COLLECTIONS_FIELDS.JOBS[3].key,
+      }),
+    shouldSkipCheck: () => false,
+    estimatedDurationSec: 3,
+  },
+};
 
+const COLLECTIONS = {
+  AMOUNT_OF_JOBS_PER_DEPARTMENT: 'AmountOfJobsPerDepartment',
+  CITIES: 'cities',
+  JOBS: 'Jobs',
+};
 
+const COLLECTIONS_FIELDS = {
+  AMOUNT_OF_JOBS_PER_DEPARTMENT: [
+    { key: 'title', type: 'TEXT' },
+    { key: 'count', type: 'NUMBER' },
+  ],
+  CITIES: [
+    { key: 'title', type: 'TEXT' },
+    { key: 'regionCode', type: 'TEXT' },
+    { key: 'city', type: 'TEXT' },
+    { key: 'location', type: 'OBJECT' },
+    { key: 'count', type: 'NUMBER' },
+    { key: 'country', type: 'TEXT' },
+    { key: 'remote', type: 'TEXT' },
+    { key: 'countryCode', type: 'TEXT' },
+    { key: 'manual', type: 'TEXT' },
+    { key: 'region', type: 'TEXT' },
+    { key: 'latitude', type: 'NUMBER' },
+    { key: 'longitude', type: 'NUMBER' },
+  ],
+  JOBS: [
+    { key: 'location', type: 'OBJECT' },
+    { key: 'postingStatus', type: 'TEXT' },
+    { key: 'country', type: 'TEXT' },
+    { key: 'department', type: 'TEXT' },
+    { key: 'language', type: 'TEXT' },
+    { key: 'jobDescription', type: 'OBJECT' },
+    { key: 'cityText', type: 'TEXT' },
+    {
+      key: 'departmentRef',
+      type: 'REFERENCE',
+      typeMetadata: { reference: { referencedCollectionId: 'AmountOfJobsPerDepartment' } },
+    },
+    {
+      key: 'city',
+      type: 'REFERENCE',
+      typeMetadata: { reference: { referencedCollectionId: 'cities' } },
+    },
+  ],
+};
 
 const TASK_TYPE = {
-    SCHEDULED: 'scheduled',
-    EVENT: 'event',
-  };
+  SCHEDULED: 'scheduled',
+  EVENT: 'event',
+};
 
-  module.exports = {
-    TASKS_NAMES,
-    TASK_TYPE,
-    TASKS,
+module.exports = {
+  TASKS_NAMES,
+  TASK_TYPE,
+  TASKS,
+  COLLECTIONS,
+  QUERY_MAX_LIMIT,
 };

package/backend/data.js

@@ -1,355 +1,338 @@
 const { items: wixData } = require('@wix/data');
 const { fetchPositionsFromSRAPI, fetchJobDescription } = require('./fetchPositionsFromSRAPI');
-const { chunkedBulkOperation } = require('./utils');
-const { createCollectionIfMissing } = require('@hisense-staging/velo-npm/backend');
-const { COLLECTIONS, COLLECTIONS_FIELDS } = require('./collectionConsts');
-const { secrets } = require("@wix/secrets");
-const { auth } = require('@wix/essentials');
+const { chunkedBulkOperation, delay, processJobsForField } = require('./utils');
+const { QUERY_MAX_LIMIT } = require('./consts');
 
 // Utility function to normalize city names
 function normalizeCityName(city) {
-    if (!city) return city;
-    // Remove accents/diacritics, trim whitespace
-    return city.normalize('NFD').replace(/\p{Diacritic}/gu, '').trim();
+  if (!city) return city;
+  // Remove accents/diacritics, trim whitespace
+  return city
+    .normalize('NFD')
+    .replace(/\p{Diacritic}/gu, '')
+    .trim();
 }
 
-async function saveDataJobsToCMS() {
-    const positions = await fetchPositionsFromSRAPI();
-    // bulk insert to jobs collection without descriptions first
-    const jobsData = positions.content.map((position) => {
-        const basicJob = {
-            _id: position.id,
-            title: position.title,
-            department: position.department.label,
-            cityText: normalizeCityName(position.location.city),
-            location: position.location,
-            country: position.location.country,
-            remote: position.location.remote,
-            language: position.language.label,
-            postingStatus: position.postingStatus,
-            jobDescription: null // Will be filled later
-        }
-        return basicJob;
-    });
-    
-    const chunkSize = 1000;
-    let totalSaved = 0;
-    const totalChunks = Math.ceil(jobsData.length / chunkSize);
-    
-    console.log(`Processing ${jobsData.length} jobs in ${totalChunks} chunks of max ${chunkSize} items each`);
-
-    await chunkedBulkOperation({
-        items: jobsData,
-        chunkSize,
+async function saveJobsDataToCMS() {
+  const positions = await fetchPositionsFromSRAPI();
+  // bulk insert to jobs collection without descriptions first
+  const jobsData = positions.content.map(position => {
+    const basicJob = {
+      _id: position.id,
+      title: position.title,
+      department: position.department.label,
+      cityText: normalizeCityName(position.location.city),
+      location: position.location,
+      country: position.location.country,
+      remote: position.location.remote,
+      language: position.language.label,
+      postingStatus: position.postingStatus,
+      jobDescription: null, // Will be filled later
+    };
+    return basicJob;
+  });
+
+  const chunkSize = 1000;
+  let totalSaved = 0;
+  const totalChunks = Math.ceil(jobsData.length / chunkSize);
+
+  console.log(
+    `Processing ${jobsData.length} jobs in ${totalChunks} chunks of max ${chunkSize} items each`
+  );
+
+  await chunkedBulkOperation({
+    items: jobsData,
+    chunkSize,
+    processChunk: async (chunk, chunkNumber) => {
+      console.log(`Saving chunk ${chunkNumber}/${totalChunks}: ${chunk.length} jobs`);
+      try {
+        const result = await wixData.bulkSave('Jobs', chunk);
+        const saved = result.inserted + result.updated || chunk.length;
+        totalSaved += saved;
+        console.log(
+          `✓ Chunk ${chunkNumber} saved successfully. Inserted: ${result.inserted}, Updated: ${result.updated}`
+        );
+      } catch (error) {
+        console.error(`✗ Error saving chunk ${chunkNumber}:`, error);
+        throw error;
+      }
+    },
+  });
+
+  console.log(`✓ All chunks processed. Total jobs saved: ${totalSaved}/${jobsData.length}`);
+}
+
+async function saveJobsDescriptionsAndLocationToCMS() {
+  console.log('🚀 Starting job descriptions update process for ALL jobs using pagination...');
+
+  try {
+    let jobsWithNoDescriptions = await getJobsWithNoDescriptions();
+    let totalUpdated = 0;
+    let totalFailed = 0;
+    let totalProcessed = 0;
+    let pageNumber = 1;
+
+    console.log(
+      `Total jobs in database without descriptions:  ${jobsWithNoDescriptions.totalCount}`
+    );
+
+    if (jobsWithNoDescriptions.totalCount === 0) {
+      console.log('No jobs found in database');
+      return { success: true, message: 'No jobs found' };
+    }
+
+    // Process all pages using hasNext() pagination
+    do {
+      const currentPageJobs = jobsWithNoDescriptions.items;
+      console.log(`\n📄 Processing page ${pageNumber} with ${currentPageJobs.length} jobs...`);
+
+      const API_CHUNK_SIZE = 80;
+      const pageChunks = Math.ceil(currentPageJobs.length / API_CHUNK_SIZE);
+
+      await chunkedBulkOperation({
+        items: currentPageJobs,
+        chunkSize: API_CHUNK_SIZE,
         processChunk: async (chunk, chunkNumber) => {
-            console.log(`Saving chunk ${chunkNumber}/${totalChunks}: ${chunk.length} jobs`);
+          console.log(`  Processing API chunk ${chunkNumber}/${pageChunks} (${chunk.length} jobs)`);
+          const chunkPromises = chunk.map(async job => {
             try {
-                const result = await wixData.bulkSave("Jobs", chunk);
-                const saved = result.inserted + result.updated || chunk.length;
-                totalSaved += saved;
-                console.log(`✓ Chunk ${chunkNumber} saved successfully. Inserted: ${result.inserted}, Updated: ${result.updated}`);
+              const jobDetails = await fetchJobDescription(job._id);
+              const jobLocation = fetchJobLocation(jobDetails);
+
+              const updatedJob = {
+                ...job,
+                locationAddress: jobLocation,
+                jobDescription: jobDetails.jobAd.sections,
+              };
+              await wixData.update(COLLECTIONS.JOBS, updatedJob);
+              return { success: true, jobId: job._id, title: job.title };
             } catch (error) {
-                console.error(`✗ Error saving chunk ${chunkNumber}:`, error);
-                throw error;
+              console.error(`    ❌ Failed to update ${job.title} (${job._id}):`, error);
+              return { success: false, jobId: job._id, title: job.title, error: error.message };
             }
-        }
-    });
-    
-    console.log(`✓ All chunks processed. Total jobs saved: ${totalSaved}/${jobsData.length}`);
+          });
+          const chunkResults = await Promise.all(chunkPromises);
+          const chunkSuccesses = chunkResults.filter(r => r.success).length;
+          const chunkFailures = chunkResults.filter(r => !r.success).length;
+          totalUpdated += chunkSuccesses;
+          totalFailed += chunkFailures;
+          totalProcessed += chunk.length;
+          console.log(
+            `  API chunk ${chunkNumber} completed: ${chunkSuccesses} success, ${chunkFailures} failed`
+          );
+          if (chunkNumber * API_CHUNK_SIZE < currentPageJobs.length) {
+            console.log('  Waiting 2 seconds before next API chunk...');
+            await delay(2000);
+          }
+        },
+      });
 
-}
+      console.log(
+        `📄 Page ${pageNumber} completed. Updated: ${totalUpdated}, Failed: ${totalFailed}, Total processed: ${totalProcessed}/${jobsWithNoDescriptions.totalCount}`
+      );
 
-async function saveJobsDescriptionsAndLocationToCMS() {
-    
-   console.log('🚀 Starting job descriptions update process for ALL jobs using pagination...');
-    
-    try {
-        let jobsWithNoDescriptions = await getJobsWithNoDescriptions();
-        let totalUpdated = 0;
-        let totalFailed = 0;
-        let totalProcessed = 0;
-        let pageNumber = 1;
-        
-        // Start with the first page query - limit to 100 jobs per page
-       // let jobsQuery = await wixData.query("Jobs").limit(300).find();
-        
-        console.log(`Total jobs in database without descriptions:  ${jobsWithNoDescriptions.totalCount}`);
-        
-        if (jobsWithNoDescriptions.totalCount === 0) {
-            console.log('No jobs found in database');
-            return { success: true, message: 'No jobs found' };
-        }
-
-        // Process all pages using hasNext() pagination
-        do {
-            const currentPageJobs = jobsWithNoDescriptions.items;
-            console.log(`\n📄 Processing page ${pageNumber} with ${currentPageJobs.length} jobs...`);
-            
-            // Process jobs in smaller chunks of 5 for API calls within each page
-            const API_CHUNK_SIZE = 80;
-            const pageChunks = Math.ceil(currentPageJobs.length / API_CHUNK_SIZE);
-            
-            await chunkedBulkOperation({
-                items: currentPageJobs,
-                chunkSize: API_CHUNK_SIZE,
-                processChunk: async (chunk, chunkNumber) => {
-                    console.log(`  Processing API chunk ${chunkNumber}/${pageChunks} (${chunk.length} jobs)`);
-                    const chunkPromises = chunk.map(async (job) => {
-                        try {
-                            //   console.log(`    Fetching description for: ${job.title} (${job._id})`);
-                            const jobDetails = await fetchJobDescription(job._id);
-                            const jobLocation = fetchJobLocation(jobDetails)
-                            
-                            const updatedJob = {
-                                ...job,
-                                locationAddress: jobLocation,
-                                jobDescription: jobDetails.jobAd.sections
-                            };
-                            await wixData.update("Jobs", updatedJob);
-                            // console.log(`    ✅ Updated description for: ${job.title}`);
-                            return { success: true, jobId: job._id, title: job.title };
-                        } catch (error) {
-                            console.error(`    ❌ Failed to update ${job.title} (${job._id}):`, error);
-                            return { success: false, jobId: job._id, title: job.title, error: error.message };
-                        }
-                    });
-                    const chunkResults = await Promise.all(chunkPromises);
-                    const chunkSuccesses = chunkResults.filter(r => r.success).length;
-                    const chunkFailures = chunkResults.filter(r => !r.success).length;
-                    totalUpdated += chunkSuccesses;
-                    totalFailed += chunkFailures;
-                    totalProcessed += chunk.length;
-                    console.log(`  API chunk ${chunkNumber} completed: ${chunkSuccesses} success, ${chunkFailures} failed`);
-                    if (chunkNumber * API_CHUNK_SIZE < currentPageJobs.length) {
-                        console.log('  Waiting 2 seconds before next API chunk...');
-                        await new Promise(resolve => setTimeout(resolve, 2000));
-                    }
-                }
-            });
-            
-            console.log(`📄 Page ${pageNumber} completed. Updated: ${totalUpdated}, Failed: ${totalFailed}, Total processed: ${totalProcessed}/${jobsWithNoDescriptions.totalCount}`);
-            
-            // Check if there are more pages and get the next page
-            if (jobsWithNoDescriptions.hasNext()) {
-                console.log('🔄 Moving to next page...');
-                jobsWithNoDescriptions = await jobsWithNoDescriptions.next();
-                pageNumber++;
-                
-                // Add a delay between pages
-                console.log('Waiting 3 seconds before next page...');
-                await new Promise(resolve => setTimeout(resolve, 3000));
-            }
-            
-        } while (jobsWithNoDescriptions.hasNext());
-        
-        console.log(`\n✅ Finished updating ALL job descriptions using pagination!`);
-        console.log(`📊 Final Results:`);
-        console.log(`   Total pages processed: ${pageNumber}`);
-        console.log(`   Total jobs processed: ${totalProcessed}`);
-        console.log(`   Total updated: ${totalUpdated}`);
-        console.log(`   Total failed: ${totalFailed}`);
-        
-        return {
-            success: true,
-            totalPages: pageNumber,
-            totalProcessed: totalProcessed,
-            totalUpdated: totalUpdated,
-            totalFailed: totalFailed,
-            message: `Successfully updated ${totalUpdated} job descriptions out of ${totalProcessed} total jobs across ${pageNumber} pages`
-        };
-        
-    } catch (error) {
-        console.error('❌ Error in updateJobDescriptions:', error);
-        throw error;
-    }
+      // Check if there are more pages and get the next page
+      if (jobsWithNoDescriptions.hasNext()) {
+        console.log('🔄 Moving to next page...');
+        jobsWithNoDescriptions = await jobsWithNoDescriptions.next();
+        pageNumber++;
+
+        // Add a delay between pages
+        console.log('Waiting 3 seconds before next page...');
+        await delay(3000);
+      }
+    } while (jobsWithNoDescriptions.hasNext());
+
+    console.log(`\n✅ Finished updating ALL job descriptions using pagination!`);
+    console.log(`📊 Final Results:`);
+    console.log(`   Total pages processed: ${pageNumber}`);
+    console.log(`   Total jobs processed: ${totalProcessed}`);
+    console.log(`   Total updated: ${totalUpdated}`);
+    console.log(`   Total failed: ${totalFailed}`);
+
+    return {
+      success: true,
+      totalPages: pageNumber,
+      totalProcessed: totalProcessed,
+      totalUpdated: totalUpdated,
+      totalFailed: totalFailed,
+      message: `Successfully updated ${totalUpdated} job descriptions out of ${totalProcessed} total jobs across ${pageNumber} pages`,
+    };
+  } catch (error) {
+    console.error('❌ Error in updateJobDescriptions:', error);
+    throw error;
+  }
 }
 
 async function aggregateJobsByFieldToCMS({ field, collection }) {
-    console.log(`counting jobs per ${field}.`);
-    let jobsPerField = {};
-    let cityLocations = {};
-    let query = wixData.query("Jobs").limit(1000);
-    let results = await query.find();
-    let page = 1;
-    do {
-        console.log(`Page ${page}: ${results.items.length} jobs.`);
-        for (const job of results.items) {
-            if (!job[field])
-                {
-                    throw new Error(`Job ${job._id} has no ${field} field`);
-                } 
-            jobsPerField[job[field]] = (jobsPerField[job[field]] || 0) + 1;
-            if (field === 'cityText' && !cityLocations[job[field]]) {
-                cityLocations[job[field]] = job.location;
-            }
-        }
-        if (results.hasNext()) {
-            results = await results.next();
-            page++;
-        }
-    } while (results.hasNext());
-    let toSave = [];
-    if (field === 'cityText') {
-         toSave = Object.entries(jobsPerField).map(([value, amount]) => {
-            const loc = cityLocations[value] || {};
-            value = normalizeCityName(value);
-
-            return {
-                title: value,
-                _id: value.replace(/\s+/g, ''),
-                count: amount,
-                location: loc,
-                countryCode: loc.countryCode,
-                country: loc.country,
-                region: loc.region,
-                city: loc.city,
-                manual: loc.manual.toString(),
-                remote: loc.remote.toString(),
-                regionCode: loc.regionCode,
-                latitude: loc.latitude,
-                longitude: loc.longitude
-            };
-        });
+  console.log(`counting jobs per ${field}.`);
+  const jobsPerField = {};
+  const cityLocations = {};
+  let results = await wixData.query(COLLECTIONS.JOBS).limit(QUERY_MAX_LIMIT).find();
+  let page = 1;
+  do {
+    console.log(`Page ${page}: ${results.items.length} jobs.`);
+    processJobsForField(results.items, field, jobsPerField, cityLocations);
+    if (results.hasNext()) {
+      results = await results.next();
+      page++;
     }
-    else{
-    // Prepare array for bulkSave
-     toSave = Object.entries(jobsPerField).map(([value, amount]) => ({
+  } while (results.hasNext());
+  let toSave = [];
+  if (field === COLLECTIONS_FIELDS.JOBS[6].key) {
+    toSave = Object.entries(jobsPerField).map(([value, amount]) => {
+      const loc = cityLocations[value] || {};
+      value = normalizeCityName(value);
+
+      return {
         title: value,
         _id: value.replace(/\s+/g, ''),
-        count: amount
+        count: amount,
+        location: loc,
+        countryCode: loc.countryCode,
+        country: loc.country,
+        region: loc.region,
+        city: loc.city,
+        manual: loc.manual.toString(),
+        remote: loc.remote.toString(),
+        regionCode: loc.regionCode,
+        latitude: loc.latitude,
+        longitude: loc.longitude,
+      };
+    });
+  } else {
+    // Prepare array for bulkSave
+    toSave = Object.entries(jobsPerField).map(([value, amount]) => ({
+      title: value,
+      _id: value.replace(/\s+/g, ''),
+      count: amount,
     }));
-    }
-    if (toSave.length === 0) {
-        console.log('No jobs found.');
-        return { success: true, message: 'No jobs to save.' };
-    }
+  }
+  if (toSave.length === 0) {
+    console.log('No jobs found.');
+    return { success: true, message: 'No jobs to save.' };
+  }
 
-    try {
-        const saveResult = await wixData.bulkSave(collection, toSave);
-        console.log(`Saved ${toSave.length} ${field} counts to ${collection}.`);
-        return { success: true, saved: toSave.length, result: saveResult };
-    } catch (err) {
-        console.error(`Error saving jobs per ${field}:`, err);
-        return { success: false, error: err.message };
-    }
+  try {
+    const saveResult = await wixData.bulkSave(collection, toSave);
+    console.log(`Saved ${toSave.length} ${field} counts to ${collection}.`);
+    return { success: true, saved: toSave.length, result: saveResult };
+  } catch (err) {
+    console.error(`Error saving jobs per ${field}:`, err);
+    return { success: false, error: err.message };
+  }
 }
 
 async function getJobsWithNoDescriptions() {
-    
-    let jobswithoutdescriptionsQuery = await wixData.query("Jobs").limit(1000).isEmpty("jobDescription").find(); // with 900 as the limit, 429 error won't happen
-    return jobswithoutdescriptionsQuery;
+  let jobswithoutdescriptionsQuery = await wixData
+    .query(COLLECTIONS.JOBS)
+    .limit(QUERY_MAX_LIMIT)
+    .isEmpty('jobDescription')
+    .find();
+  return jobswithoutdescriptionsQuery;
 }
 
-async function referenceJobsToField({
-    referenceField,      // e.g., "city" or "department"
-    sourceCollection,    // e.g., "cities" or "departments"
-    jobField,            // e.g., "cityText" or "department"
-    
-}) {
-    // Fetch all source items (cities or departments)
-    const sources = await wixData.query(sourceCollection).limit(1000).find();
-    const sourceMap = {};
-    for (const item of sources.items) {
-        sourceMap[item.title] = item._id;
-    }
-
-    // Fetch all jobs
-    let jobsResults = await wixData.query("Jobs").limit(1000).find();
-    let jobsToUpdate = [];
-    console.log('jobsResults',jobsResults.items);
+/**
+ * @param {Object} params
+ * @param {"city"|"departmentref"} params.referenceField
+ * @param {"cities"|"AmountOfJobsPerDepartment"} params.sourceCollection
+ * @param {"cityText"|"department"} params.jobField
+ */
+async function referenceJobsToField({ referenceField, sourceCollection, jobField }) {
+  // Fetch all source items (cities or departments)
+  const sources = await wixData.query(sourceCollection).limit(QUERY_MAX_LIMIT).find();
+  const sourceMap = {};
+  for (const item of sources.items) {
+    sourceMap[item.title] = item._id;
+  }
 
-    do {
-        for (const job of jobsResults.items) {
-            const refId = sourceMap[job[jobField]];
-            if (refId) {
-                jobsToUpdate.push({
-                    ...job,
-                    [referenceField]: refId
-                });
-            }
-        }
-        if (jobsResults.hasNext()) {
-            jobsResults = await jobsResults.next();
-        } else {
-            break;
-        }
-    } while (true);
-
-    // Remove system fields that cannot be updated
-    jobsToUpdate = jobsToUpdate.map(job => {
-        const { _createdDate, _updatedDate, ...rest } = job;
-        return rest;
-    });
+  // Fetch all jobs
+  let jobsResults = await wixData.query('Jobs').limit(QUERY_MAX_LIMIT).find();
+  let jobsToUpdate = [];
+  console.log('jobsResults', jobsResults.items);
 
-    // Bulk update in chunks of 1000
-    const chunkSize = 1000;
-    await chunkedBulkOperation({
-        items: jobsToUpdate,
-        chunkSize,
-        processChunk: async (chunk) => {
-            await wixData.bulkUpdate("Jobs", chunk);
-        }
-    });
+  let maxIterations = 3000; // Prevent infinite loop
+  let iterationCount = 0;
 
-    return { success: true, updated: jobsToUpdate.length };
-}
+  do {
+    for (const job of jobsResults.items) {
+      const refId = sourceMap[job[jobField]];
+      if (refId) {
+        jobsToUpdate.push({
+          ...job,
+          [referenceField]: refId,
+        });
+      }
+    }
 
-function fetchJobLocation(jobDetails) {
-    const isZeroLocation = jobDetails.location.latitude === "0.0000" && jobDetails.location.longitude === "0.0000";
-    const jobLocation = {
-        location: isZeroLocation ? {} : {
-            latitude: parseFloat(jobDetails.location.latitude),
-            longitude: parseFloat(jobDetails.location.longitude)
-        },
-        city: jobDetails.location.city,
-        country: jobDetails.location.country,
-        formatted: [
-            jobDetails.location.city,
-            jobDetails.location.region,
-            jobDetails.location.regionCode,
-            jobDetails.location.country
-        ].filter(Boolean).join(', '),
-        streetAddress: {},
-        subdivision: "",
-        postalCode: ""
-    };
-    return jobLocation;
+    iterationCount++;
+    if (iterationCount >= maxIterations) {
+      console.warn(`Max iterations (${maxIterations}) reached. Stopping to prevent infinite loop.`);
+      break;
+    }
 
-}
+    try {
+      if (jobsResults.hasNext()) {
+        jobsResults = await jobsResults.next();
+      } else {
+        break;
+      }
+    } catch (error) {
+      console.log(' Error in referenceJobsToField:', error);
+      throw error;
+    }
+  } while (true);
 
+  // Remove system fields that cannot be updated
+  jobsToUpdate = jobsToUpdate.map(job => {
+    const { _createdDate, _updatedDate, ...rest } = job;
+    return rest;
+  });
 
+  // Bulk update in chunks of 1000
+  const chunkSize = 1000;
+  await chunkedBulkOperation({
+    items: jobsToUpdate,
+    chunkSize,
+    processChunk: async chunk => {
+      await wixData.bulkUpdate('Jobs', chunk);
+    },
+  });
 
-function getSmartToken() {
-  const elevatedGetSecretValue = auth.elevate(secrets.getSecretValue);
-  return elevatedGetSecretValue("x-smarttoken")
-    .then((secret) => {
-      return secret;
-    })
-    .catch((error) => {
-      console.error(error);
-    });
+  return { success: true, updated: jobsToUpdate.length };
 }
 
-
-async function createApiKeyCollectionAndFillIt() {
-    console.log("Creating ApiKey collection and filling it with the smart token");
-    await createCollectionIfMissing(COLLECTIONS.API_KEY, COLLECTIONS_FIELDS.API_KEY,null,'singleItem');
-    console.log("Getting the smart token");
-    const token = await getSmartToken();
-    console.log("token is :  ", token);
-    console.log("Inserting the smart token into the ApiKey collection");
-    await wixData.insert("ApiKey", {
-        token: token.value
-    });
-
-    console.log("Smart token inserted into the ApiKey collection");
+function fetchJobLocation(jobDetails) {
+  console.log('jobDetails.location is ', jobDetails.location);
+  const isZeroLocation =
+    jobDetails.location.latitude === '0.0000' && jobDetails.location.longitude === '0.0000';
+  const jobLocation = {
+    location: isZeroLocation
+      ? {}
+      : {
+          latitude: parseFloat(jobDetails.location.latitude),
+          longitude: parseFloat(jobDetails.location.longitude),
+        },
+    city: jobDetails.location.city,
+    country: jobDetails.location.country,
+    formatted: [
+      jobDetails.location.city,
+      jobDetails.location.region,
+      jobDetails.location.regionCode,
+      jobDetails.location.country,
+    ]
+      .filter(Boolean)
+      .join(', '),
+    streetAddress: {},
+    subdivision: '',
+    postalCode: '',
+  };
+  return jobLocation;
 }
 
-
 module.exports = {
-    saveDataJobsToCMS,
-    saveJobsDescriptionsAndLocationToCMS,
-    aggregateJobsByFieldToCMS,
-    referenceJobsToField,
-    createApiKeyCollectionAndFillIt,
-};
+  saveJobsDataToCMS,
+  saveJobsDescriptionsAndLocationToCMS,
+  aggregateJobsByFieldToCMS,
+  referenceJobsToField,
+};

package/backend/fetchPositionsFromSRAPI.js

@@ -1,21 +1,20 @@
 const { fetch } = require('wix-fetch');
-const { items: wixData } = require('@wix/data');
 
-async function makeSmartRecruitersRequest(path,token) {
+async function makeSmartRecruitersRequest(path) {
   // const baseUrl = 'https://api.smartrecruiters.com'; // PROD
   const baseUrl = 'https://aoxley54.wixstudio.com/external-template/_functions'; // TEST
-    const fullUrl = `${baseUrl}${path}`;
-  
-    //console.log(`Making request to: ${fullUrl}`);
+  const fullUrl = `${baseUrl}${path}`;
+  //console.log(`Making request to: ${fullUrl}`);
   try {
     const response = await fetch(fullUrl, {
       method: 'GET',
       headers: {
         'Accept-Language': 'en',
-        'accept': 'application/json',
-        'x-smarttoken': token,
-        'Cookie': 'AWSALB=GYltFw3fLKortMxHR5vIOT1CuUROyhWNIX/qL8ZnPl1/8mhOcnIsBKYslzmNJPEzSy/jvNbO+6tXpH8yqcpQJagYt57MhbKlLqTSzoNq1G/w7TjOxPGR3UTdXW0d; AWSALBCORS=GYltFw3fLKortMxHR5vIOT1CuUROyhWNIX/qL8ZnPl1/8mhOcnIsBKYslzmNJPEzSy/jvNbO+6tXpH8yqcpQJagYt57MhbKlLqTSzoNq1G/w7TjOxPGR3UTdXW0d'
-      }
+        accept: 'application/json',
+        'x-smarttoken': 'DCRA1-1d30ea5fe9be42d9b9ae94ff933ebef5',
+        Cookie:
+          'AWSALB=GYltFw3fLKortMxHR5vIOT1CuUROyhWNIX/qL8ZnPl1/8mhOcnIsBKYslzmNJPEzSy/jvNbO+6tXpH8yqcpQJagYt57MhbKlLqTSzoNq1G/w7TjOxPGR3UTdXW0d; AWSALBCORS=GYltFw3fLKortMxHR5vIOT1CuUROyhWNIX/qL8ZnPl1/8mhOcnIsBKYslzmNJPEzSy/jvNbO+6tXpH8yqcpQJagYt57MhbKlLqTSzoNq1G/w7TjOxPGR3UTdXW0d',
+      },
     });
 
     if (response.ok) {
@@ -34,53 +33,51 @@ async function fetchPositionsFromSRAPI() {
   let allPositions = [];
   let totalFound = 0;
   let nextPageId = null; // Start with no page ID for the first request
-  let pageCount = 0;
-  const MAX_PAGES = 30 // Safety limit to prevent infinite loops
-  const token = await getSmartTokenFromCMS();
+  let page = 0;
+  const MAX_PAGES = 30; // Safety limit to prevent infinite loops
 
   console.log('Starting to fetch all positions with pagination...');
 
   do {
     try {
-      pageCount++;
-      
+      page++;
+
       // Build the API path - first request has no page parameter, subsequent use nextPageId
       let apiPath = '/jobs?limit=50';
       if (nextPageId) {
         apiPath += `&nextPageId=${nextPageId}`;
       }
-      
-      console.log(`Fetching page ${pageCount} with path: ${apiPath}`);
-      const response = await makeSmartRecruitersRequest(apiPath,token);
-      
+
+      console.log(`Fetching page ${page} with path: ${apiPath}`);
+      const response = await makeSmartRecruitersRequest(apiPath);
+
       // Add positions from this page to our collection
       if (response.content && Array.isArray(response.content)) {
         allPositions = allPositions.concat(response.content);
-        console.log(`Page ${pageCount}: Found ${response.content.length} positions`);
+        console.log(`Page ${page}: Found ${response.content.length} positions`);
       }
-      
+
       // Update total count from first response
-      if (pageCount === 1) {
+      if (page === 1) {
         totalFound = response.totalFound || 0;
         console.log(`Total positions available: ${totalFound}`);
       }
-      
+
       // Get the nextPageId for the next iteration
-      nextPageId = response.nextPageId && response.nextPageId !== "" ? response.nextPageId : null;
-      
+      nextPageId = response.nextPageId && response.nextPageId !== '' ? response.nextPageId : null;
+
       if (nextPageId) {
         console.log(`Next page ID: ${nextPageId}`);
       } else {
         console.log('No more pages to fetch');
       }
-      
     } catch (error) {
-      console.error(`Error fetching page ${pageCount}:`, error);
+      console.error(`Error fetching page ${page}:`, error);
       throw error;
     }
-    
+
     // Safety check to prevent infinite loops
-    if (pageCount >= MAX_PAGES) {
+    if (page >= MAX_PAGES) {
       console.warn(`Reached maximum page limit of ${MAX_PAGES}. Stopping pagination.`);
       break;
     }
@@ -89,12 +86,12 @@ async function fetchPositionsFromSRAPI() {
   console.log(`Finished fetching all pages. Total positions collected: ${allPositions.length}`);
 
   // Return response in the same format as before, but with all positions
-  const result =  {
+  const result = {
     totalFound: totalFound,
     offset: 0,
     limit: allPositions.length,
-    nextPageId: "", // Always empty since we've fetched everything
-    content: allPositions
+    nextPageId: '', // Always empty since we've fetched everything
+    content: allPositions,
   };
 
   const amountOfUniqueJobs = new Set(allPositions.map(job => job.id)).size;
@@ -111,17 +108,7 @@ async function fetchJobDescription(jobId) {
   return await makeSmartRecruitersRequest(`/jobs/${jobId}`);
 }
 
-async function getSmartTokenFromCMS() {
-  const result = await wixData.query("ApiKey").limit(1).find();
-  if (result.items.length > 0) {
-      return result.items[0].token; // This is your string token
-  } else {
-      return null; // No token found
-  }
-}
-
-
 module.exports = {
-    fetchPositionsFromSRAPI,
-    fetchJobDescription,
+  fetchPositionsFromSRAPI,
+  fetchJobDescription,
 };

package/backend/index.js

@@ -1,7 +1,6 @@
 module.exports = {
-    ...require('./utils'),
-    ...require('./queries'),
-    ...require('./fetchPositionsFromSRAPI'),
-    ...require('./consts'),
-  };
-  
+  ...require('./utils'),
+  ...require('./queries'),
+  ...require('./fetchPositionsFromSRAPI'),
+  ...require('./consts'),
+};

package/backend/queries.js

@@ -1,19 +1,23 @@
-const { COLLECTIONS } = require('./consts');
-const { items } = require('@wix/data');
-
+const { COLLECTIONS,QUERY_MAX_LIMIT} = require('./consts');
+const { items: wixData } = require('@wix/data');
 
 async function getAllPositions() {
-    return await items.query(COLLECTIONS.JOBS).find().items;
-    
+  return wixData
+    .query(COLLECTIONS.JOBS)
+    .limit(QUERY_MAX_LIMIT)
+    .find()
+    .then(result => result.items);
 }
 
 async function getPositionsByField(field, value) {
-    return await items.query(COLLECTIONS.JOBS).where(field, value).find().items;
-    
+  return wixData
+    .query(COLLECTIONS.JOBS)
+    .eq(field, value)
+    .find()
+    .then(result => result.items);
 }
 
-
 module.exports = {
-    getAllPositions,
-    getPositionsByField,
-};
+  getAllPositions,
+  getPositionsByField,
+};

package/backend/utils.js

@@ -1,11 +1,28 @@
 async function chunkedBulkOperation({ items, chunkSize, processChunk }) {
-    for (let i = 0; i < items.length; i += chunkSize) {
-        const chunk = items.slice(i, i + chunkSize);
-        await processChunk(chunk, Math.floor(i / chunkSize) + 1);
-    }
+  for (let i = 0; i < items.length; i += chunkSize) {
+    const chunk = items.slice(i, i + chunkSize);
+    await processChunk(chunk, Math.floor(i / chunkSize) + 1);
+  }
+}
+
+async function delay(ms) {
+  await new Promise(resolve => setTimeout(resolve, ms));
 }
 
+function processJobsForField(jobs, field, jobsPerField, cityLocations) {
+  for (const job of jobs) {
+    if (!job[field]) {
+      throw new Error(`Job ${job._id} has no ${field} field`);
+    }
+    jobsPerField[job[field]] = (jobsPerField[job[field]] || 0) + 1;
+    if (field === 'cityText' && !cityLocations[job[field]]) {
+      cityLocations[job[field]] = job.location;
+    }
+  }
+}
 
 module.exports = {
-    chunkedBulkOperation,
-};
+  chunkedBulkOperation,
+  delay,
+  processJobsForField,
+};

package/index.js

@@ -1,6 +1,5 @@
 module.exports = {
-    ...require('./public'),
-    ...require('./pages'),
-    ...require('./backend'),
-  };
-  
+  ...require('./public'),
+  ...require('./pages'),
+  ...require('./backend'),
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sr-npm",
-  "version": "1.7.34",
+  "version": "1.7.36",
   "description": "",
   "main": "index.js",
   "scripts": {
@@ -17,9 +17,10 @@
   },
   "homepage": "https://github.com/psdevteamenterprise/sr-npm#readme",
   "dependencies": {
-    "@hisense-staging/velo-npm": "1.7.250",
-    "@wix/data": "^1.0.211",
-    "@wix/essentials": "^0.1.24",
-    "@wix/secrets": "1.0.53"
+    "@hisense-staging/velo-npm": "1.7.249",
+    "@wix/data": "^1.0.211"
+  },
+  "devDependencies": {
+    "prettier": "^3.6.2"
   }
 }

package/backend/collectionConsts.js

@@ -1,48 +0,0 @@
-const COLLECTIONS = {
-    AMOUNT_OF_JOBS_PER_DEPARTMENT: 'AmountOfJobsPerDepartment',
-    CITIES: 'cities',
-    JOBS: 'Jobs',
-    API_KEY: 'ApiKey',
-}
-
-const COLLECTIONS_FIELDS = {
-    AMOUNT_OF_JOBS_PER_DEPARTMENT: [
-      {key:'title', type: 'TEXT'},
-      { key: 'count', type: 'NUMBER' },
-    ],
-    CITIES: [
-      {key:'title', type: 'TEXT'},
-      { key: 'regionCode', type: 'TEXT' },
-      { key: 'city', type: 'TEXT' },
-      {key:'location', type: 'OBJECT'},
-      {key:'count', type: 'NUMBER'},
-      {key:'country', type: 'TEXT'},
-      {key:'remote', type: 'TEXT'},
-      {key:'countryCode', type: 'TEXT'},
-      {key:'manual', type: 'TEXT'},      
-      {key:'region', type: 'TEXT'},
-      {key:'latitude', type: 'NUMBER'},
-      {key:'longitude', type: 'NUMBER'},
-    ],
-    JOBS: [
-        {key:'location', type: 'OBJECT'},
-        {key:'postingStatus', type: 'TEXT'},
-        {key:'country', type: 'TEXT'},
-        {key:'department', type: 'TEXT'},
-        {key:'language', type: 'TEXT'},
-        {key:'jobDescription', type: 'OBJECT'},  
-        {key:'cityText', type: 'TEXT'},         
-        {key:'departmentref', type: 'REFERENCE', typeMetadata: { reference: { referencedCollectionId: 'AmountOfJobsPerDepartment' } } },
-        {key:'city', type: 'REFERENCE', typeMetadata: { reference: { referencedCollectionId: 'cities' } } },
-    ],  
-    API_KEY: [
-      {key:'token', type: 'TEXT'},
-    ],
-  };
-
-
-
-  module.exports = {
-    COLLECTIONS,
-    COLLECTIONS_FIELDS,
-};