sr-npm 1.0.5 → 1.0.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/backend/consts.js +139 -2
- package/backend/data.js +288 -0
- package/backend/fetchPositionsFromSRAPI.js +109 -0
- package/backend/index.js +1 -1
- package/backend/queries.js +0 -1
- package/backend/utils.js +6 -0
- package/package.json +2 -3
- package/public/index.js +0 -4
- package/backend/elevated-modules.js +0 -17
- package/public/utils.js +0 -9
package/backend/consts.js
CHANGED
|
@@ -1,5 +1,142 @@
|
|
|
1
|
-
|
|
1
|
+
import {saveDataJobsToCMS,saveJobsDescriptionsToCMS,aggregateJobsByFieldToCMS,referenceJobsToField} from './data';
|
|
2
|
+
const { createCollectionIfMissing } = require('@hisense-staging/velo-npm/backend');
|
|
3
|
+
export const TASKS_NAMES = {
|
|
4
|
+
SYNC_JOBS: 'syncJobsFromSRAPIToCMS',
|
|
5
|
+
INSERT_JOBS_TO_CMS: 'insertJobsToCMS',
|
|
6
|
+
INSERT_JOBS_DESCRIPTIONS_TO_CMS: 'insertJobsDescriptionsToCMS',
|
|
7
|
+
FILL_JOBS_PER_CITY_COLLECTION: 'fillJobsPerCityCollection',
|
|
8
|
+
FILL_JOBS_PER_DEPARTMENT_COLLECTION: 'fillJobsPerDepartmentCollection',
|
|
9
|
+
REFERENCE_JOBS_TO_LOCATIONS: 'referenceJobsToLocations',
|
|
10
|
+
REFERENCE_JOBS_TO_DEPARTMENT: 'referenceJobsToDepartment',
|
|
11
|
+
CREATE_JOBS_COLLECTION: 'createJobsCollection',
|
|
12
|
+
CREATE_CITIES_COLLECTION: 'createCitiesCollection',
|
|
13
|
+
CREATE_AMOUNT_OF_JOBS_PER_DEPARTMENT_COLLECTION: 'createAmountOfJobsPerDepartmentCollection',
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
export const TASKS = {
|
|
18
|
+
[TASKS_NAMES.SYNC_JOBS]: {
|
|
19
|
+
name: TASKS_NAMES.SYNC_JOBS,
|
|
20
|
+
childTasks: [
|
|
21
|
+
{ name: TASKS_NAMES.CREATE_JOBS_COLLECTION },
|
|
22
|
+
{ name: TASKS_NAMES.CREATE_CITIES_COLLECTION },
|
|
23
|
+
{name: TASKS_NAMES.CREATE_AMOUNT_OF_JOBS_PER_DEPARTMENT_COLLECTION},
|
|
24
|
+
{ name: TASKS_NAMES.INSERT_JOBS_TO_CMS },
|
|
25
|
+
{ name: TASKS_NAMES.INSERT_JOBS_DESCRIPTIONS_TO_CMS },
|
|
26
|
+
{ name: TASKS_NAMES.FILL_JOBS_PER_CITY_COLLECTION },
|
|
27
|
+
{ name: TASKS_NAMES.FILL_JOBS_PER_DEPARTMENT_COLLECTION },
|
|
28
|
+
{ name: TASKS_NAMES.REFERENCE_JOBS_TO_LOCATIONS },
|
|
29
|
+
{ name: TASKS_NAMES.REFERENCE_JOBS_TO_DEPARTMENT },
|
|
30
|
+
],
|
|
31
|
+
scheduleChildrenSequentially: true,
|
|
32
|
+
estimatedDurationSec: 30,
|
|
33
|
+
},
|
|
34
|
+
[TASKS_NAMES.CREATE_JOBS_COLLECTION]: {
|
|
35
|
+
name: TASKS_NAMES.CREATE_JOBS_COLLECTION,
|
|
36
|
+
getIdentifier:()=> "SHOULD_NEVER_SKIP",
|
|
37
|
+
process:()=>createCollectionIfMissing(COLLECTIONS.JOBS, COLLECTIONS_FIELDS.JOBS),
|
|
38
|
+
shouldSkipCheck:()=>false,
|
|
39
|
+
estimatedDurationSec:3
|
|
40
|
+
},
|
|
41
|
+
[TASKS_NAMES.CREATE_CITIES_COLLECTION]: {
|
|
42
|
+
name: TASKS_NAMES.CREATE_CITIES_COLLECTION,
|
|
43
|
+
getIdentifier:()=> "SHOULD_NEVER_SKIP",
|
|
44
|
+
process:()=>createCollectionIfMissing(COLLECTIONS.CITIES, COLLECTIONS_FIELDS.CITIES),
|
|
45
|
+
shouldSkipCheck:()=>false,
|
|
46
|
+
estimatedDurationSec:3
|
|
47
|
+
},
|
|
48
|
+
[TASKS_NAMES.CREATE_AMOUNT_OF_JOBS_PER_DEPARTMENT_COLLECTION]: {
|
|
49
|
+
name: TASKS_NAMES.CREATE_AMOUNT_OF_JOBS_PER_DEPARTMENT_COLLECTION,
|
|
50
|
+
getIdentifier:()=> "SHOULD_NEVER_SKIP",
|
|
51
|
+
process:()=>createCollectionIfMissing(COLLECTIONS.AMOUNT_OF_JOBS_PER_DEPARTMENT, COLLECTIONS_FIELDS.AMOUNT_OF_JOBS_PER_DEPARTMENT),
|
|
52
|
+
shouldSkipCheck:()=>false,
|
|
53
|
+
estimatedDurationSec:3
|
|
54
|
+
},
|
|
55
|
+
[TASKS_NAMES.INSERT_JOBS_TO_CMS]: {
|
|
56
|
+
name: TASKS_NAMES.INSERT_JOBS_TO_CMS,
|
|
57
|
+
getIdentifier:()=> "SHOULD_NEVER_SKIP",
|
|
58
|
+
process:saveDataJobsToCMS,
|
|
59
|
+
shouldSkipCheck:()=>false,
|
|
60
|
+
estimatedDurationSec:20
|
|
61
|
+
},
|
|
62
|
+
[TASKS_NAMES.INSERT_JOBS_DESCRIPTIONS_TO_CMS]: {
|
|
63
|
+
name: TASKS_NAMES.INSERT_JOBS_DESCRIPTIONS_TO_CMS,
|
|
64
|
+
getIdentifier:()=> "SHOULD_NEVER_SKIP",
|
|
65
|
+
process:saveJobsDescriptionsToCMS,
|
|
66
|
+
shouldSkipCheck:()=>false,
|
|
67
|
+
estimatedDurationSec:20
|
|
68
|
+
},
|
|
69
|
+
[TASKS_NAMES.FILL_JOBS_PER_CITY_COLLECTION]: {
|
|
70
|
+
name: TASKS_NAMES.FILL_JOBS_PER_CITY_COLLECTION,
|
|
71
|
+
getIdentifier:()=> "SHOULD_NEVER_SKIP",
|
|
72
|
+
process:()=>aggregateJobsByFieldToCMS({ field: 'cityText', collection: 'cities' }),
|
|
73
|
+
shouldSkipCheck:()=>false,
|
|
74
|
+
estimatedDurationSec:3
|
|
75
|
+
},
|
|
76
|
+
[TASKS_NAMES.FILL_JOBS_PER_DEPARTMENT_COLLECTION]: {
|
|
77
|
+
name: TASKS_NAMES.FILL_JOBS_PER_DEPARTMENT_COLLECTION,
|
|
78
|
+
getIdentifier:()=> "SHOULD_NEVER_SKIP",
|
|
79
|
+
process:()=>aggregateJobsByFieldToCMS({ field: 'department', collection: 'AmountOfJobsPerDepartment' }),
|
|
80
|
+
shouldSkipCheck:()=>false,
|
|
81
|
+
estimatedDurationSec:3
|
|
82
|
+
},
|
|
83
|
+
[TASKS_NAMES.REFERENCE_JOBS_TO_LOCATIONS]: {
|
|
84
|
+
name: TASKS_NAMES.REFERENCE_JOBS_TO_LOCATIONS,
|
|
85
|
+
getIdentifier:()=> "SHOULD_NEVER_SKIP",
|
|
86
|
+
process:()=>referenceJobsToField({ referenceField: 'city', sourceCollection: 'cities', jobField: 'cityText' }),
|
|
87
|
+
shouldSkipCheck:()=>false,
|
|
88
|
+
estimatedDurationSec:3
|
|
89
|
+
},
|
|
90
|
+
[TASKS_NAMES.REFERENCE_JOBS_TO_DEPARTMENT]: {
|
|
91
|
+
name: TASKS_NAMES.REFERENCE_JOBS_TO_DEPARTMENT,
|
|
92
|
+
getIdentifier:()=> "SHOULD_NEVER_SKIP",
|
|
93
|
+
process:()=>referenceJobsToField({ referenceField: 'departmentref', sourceCollection: 'AmountOfJobsPerDepartment', jobField: 'department' }),
|
|
94
|
+
shouldSkipCheck:()=>false,
|
|
95
|
+
estimatedDurationSec:3
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
const COLLECTIONS = {
|
|
2
100
|
AMOUNT_OF_JOBS_PER_DEPARTMENT: 'AmountOfJobsPerDepartment',
|
|
3
101
|
CITIES: 'cities',
|
|
4
102
|
JOBS: 'Jobs',
|
|
5
|
-
}
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
const COLLECTIONS_FIELDS = {
|
|
106
|
+
AMOUNT_OF_JOBS_PER_DEPARTMENT: [
|
|
107
|
+
{key:'title', type: 'TEXT'},
|
|
108
|
+
{ key: 'count', type: 'NUMBER' },
|
|
109
|
+
],
|
|
110
|
+
CITIES: [
|
|
111
|
+
{key:'title', type: 'TEXT'},
|
|
112
|
+
{ key: 'regionCode', type: 'TEXT' },
|
|
113
|
+
{ key: 'city', type: 'TEXT' },
|
|
114
|
+
{key:'location', type: 'OBJECT'},
|
|
115
|
+
{key:'count', type: 'NUMBER'},
|
|
116
|
+
{key:'country', type: 'TEXT'},
|
|
117
|
+
{key:'remote', type: 'TEXT'},
|
|
118
|
+
{key:'countryCode', type: 'TEXT'},
|
|
119
|
+
{key:'manual', type: 'TEXT'},
|
|
120
|
+
{key:'region', type: 'TEXT'},
|
|
121
|
+
{key:'latitude', type: 'NUMBER'},
|
|
122
|
+
{key:'longitude', type: 'NUMBER'},
|
|
123
|
+
],
|
|
124
|
+
JOBS: [
|
|
125
|
+
{key:'location', type: 'OBJECT'},
|
|
126
|
+
{key:'postingStatus', type: 'TEXT'},
|
|
127
|
+
{key:'country', type: 'TEXT'},
|
|
128
|
+
{key:'department', type: 'TEXT'},
|
|
129
|
+
{key:'language', type: 'TEXT'},
|
|
130
|
+
{key:'jobDescription', type: 'OBJECT'},
|
|
131
|
+
{key:'cityText', type: 'TEXT'},
|
|
132
|
+
{key:'departmentref', type: 'REFERENCE', typeMetadata: { reference: { referencedCollectionId: 'AmountOfJobsPerDepartment' } } },
|
|
133
|
+
{key:'city', type: 'REFERENCE', typeMetadata: { reference: { referencedCollectionId: 'cities' } } },
|
|
134
|
+
],
|
|
135
|
+
|
|
136
|
+
};
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
export const TASK_TYPE = {
|
|
140
|
+
SCHEDULED: 'scheduled',
|
|
141
|
+
EVENT: 'event',
|
|
142
|
+
};
|
package/backend/data.js
ADDED
|
@@ -0,0 +1,288 @@
|
|
|
1
|
+
const { items: wixData } = require('@wix/data');
|
|
2
|
+
import { fetchPositionsFromSRAPI, fetchJobDescription } from './fetchPositionsFromSRAPI';
|
|
3
|
+
import { chunkedBulkOperation } from './utils';
|
|
4
|
+
|
|
5
|
+
// Utility function to normalize city names
|
|
6
|
+
function normalizeCityName(city) {
|
|
7
|
+
if (!city) return city;
|
|
8
|
+
// Remove accents/diacritics, trim whitespace
|
|
9
|
+
return city.normalize('NFD').replace(/\p{Diacritic}/gu, '').trim();
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
export async function saveDataJobsToCMS() {
|
|
13
|
+
const positions = await fetchPositionsFromSRAPI();
|
|
14
|
+
// bulk insert to jobs collection without descriptions first
|
|
15
|
+
const jobsData = positions.content.map((position) => {
|
|
16
|
+
const basicJob = {
|
|
17
|
+
_id: position.id,
|
|
18
|
+
title: position.title,
|
|
19
|
+
department: position.department.label,
|
|
20
|
+
cityText: normalizeCityName(position.location.city),
|
|
21
|
+
location: position.location,
|
|
22
|
+
country: position.location.country,
|
|
23
|
+
remote: position.location.remote,
|
|
24
|
+
language: position.language.label,
|
|
25
|
+
postingStatus: position.postingStatus,
|
|
26
|
+
jobDescription: null // Will be filled later
|
|
27
|
+
}
|
|
28
|
+
return basicJob;
|
|
29
|
+
});
|
|
30
|
+
|
|
31
|
+
const chunkSize = 1000;
|
|
32
|
+
let totalSaved = 0;
|
|
33
|
+
const totalChunks = Math.ceil(jobsData.length / chunkSize);
|
|
34
|
+
|
|
35
|
+
console.log(`Processing ${jobsData.length} jobs in ${totalChunks} chunks of max ${chunkSize} items each`);
|
|
36
|
+
|
|
37
|
+
await chunkedBulkOperation({
|
|
38
|
+
items: jobsData,
|
|
39
|
+
chunkSize,
|
|
40
|
+
processChunk: async (chunk, chunkNumber) => {
|
|
41
|
+
console.log(`Saving chunk ${chunkNumber}/${totalChunks}: ${chunk.length} jobs`);
|
|
42
|
+
try {
|
|
43
|
+
const result = await wixData.bulkSave("Jobs", chunk);
|
|
44
|
+
const saved = result.inserted + result.updated || chunk.length;
|
|
45
|
+
totalSaved += saved;
|
|
46
|
+
console.log(`✓ Chunk ${chunkNumber} saved successfully. Inserted: ${result.inserted}, Updated: ${result.updated}`);
|
|
47
|
+
} catch (error) {
|
|
48
|
+
console.error(`✗ Error saving chunk ${chunkNumber}:`, error);
|
|
49
|
+
throw error;
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
});
|
|
53
|
+
|
|
54
|
+
console.log(`✓ All chunks processed. Total jobs saved: ${totalSaved}/${jobsData.length}`);
|
|
55
|
+
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
export async function saveJobsDescriptionsToCMS() {
|
|
59
|
+
|
|
60
|
+
console.log('🚀 Starting job descriptions update process for ALL jobs using pagination...');
|
|
61
|
+
|
|
62
|
+
try {
|
|
63
|
+
let jobsWithNoDescriptions = await getJobsWithNoDescriptions();
|
|
64
|
+
let totalUpdated = 0;
|
|
65
|
+
let totalFailed = 0;
|
|
66
|
+
let totalProcessed = 0;
|
|
67
|
+
let pageNumber = 1;
|
|
68
|
+
|
|
69
|
+
// Start with the first page query - limit to 100 jobs per page
|
|
70
|
+
// let jobsQuery = await wixData.query("Jobs").limit(300).find();
|
|
71
|
+
|
|
72
|
+
console.log(`Total jobs in database without descriptions: ${jobsWithNoDescriptions.totalCount}`);
|
|
73
|
+
|
|
74
|
+
if (jobsWithNoDescriptions.totalCount === 0) {
|
|
75
|
+
console.log('No jobs found in database');
|
|
76
|
+
return { success: true, message: 'No jobs found' };
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
// Process all pages using hasNext() pagination
|
|
80
|
+
do {
|
|
81
|
+
const currentPageJobs = jobsWithNoDescriptions.items;
|
|
82
|
+
console.log(`\n📄 Processing page ${pageNumber} with ${currentPageJobs.length} jobs...`);
|
|
83
|
+
|
|
84
|
+
// Process jobs in smaller chunks of 5 for API calls within each page
|
|
85
|
+
const API_CHUNK_SIZE = 80;
|
|
86
|
+
const pageChunks = Math.ceil(currentPageJobs.length / API_CHUNK_SIZE);
|
|
87
|
+
|
|
88
|
+
await chunkedBulkOperation({
|
|
89
|
+
items: currentPageJobs,
|
|
90
|
+
chunkSize: API_CHUNK_SIZE,
|
|
91
|
+
processChunk: async (chunk, chunkNumber) => {
|
|
92
|
+
console.log(` Processing API chunk ${chunkNumber}/${pageChunks} (${chunk.length} jobs)`);
|
|
93
|
+
const chunkPromises = chunk.map(async (job) => {
|
|
94
|
+
try {
|
|
95
|
+
// console.log(` Fetching description for: ${job.title} (${job._id})`);
|
|
96
|
+
const jobDetails = await fetchJobDescription(job._id);
|
|
97
|
+
const updatedJob = {
|
|
98
|
+
...job,
|
|
99
|
+
jobDescription: jobDetails.jobAd.sections
|
|
100
|
+
};
|
|
101
|
+
await wixData.update("Jobs", updatedJob);
|
|
102
|
+
// console.log(` ✅ Updated description for: ${job.title}`);
|
|
103
|
+
return { success: true, jobId: job._id, title: job.title };
|
|
104
|
+
} catch (error) {
|
|
105
|
+
console.error(` ❌ Failed to update ${job.title} (${job._id}):`, error);
|
|
106
|
+
return { success: false, jobId: job._id, title: job.title, error: error.message };
|
|
107
|
+
}
|
|
108
|
+
});
|
|
109
|
+
const chunkResults = await Promise.all(chunkPromises);
|
|
110
|
+
const chunkSuccesses = chunkResults.filter(r => r.success).length;
|
|
111
|
+
const chunkFailures = chunkResults.filter(r => !r.success).length;
|
|
112
|
+
totalUpdated += chunkSuccesses;
|
|
113
|
+
totalFailed += chunkFailures;
|
|
114
|
+
totalProcessed += chunk.length;
|
|
115
|
+
console.log(` API chunk ${chunkNumber} completed: ${chunkSuccesses} success, ${chunkFailures} failed`);
|
|
116
|
+
if (chunkNumber * API_CHUNK_SIZE < currentPageJobs.length) {
|
|
117
|
+
console.log(' Waiting 2 seconds before next API chunk...');
|
|
118
|
+
await new Promise(resolve => setTimeout(resolve, 2000));
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
});
|
|
122
|
+
|
|
123
|
+
console.log(`📄 Page ${pageNumber} completed. Updated: ${totalUpdated}, Failed: ${totalFailed}, Total processed: ${totalProcessed}/${jobsWithNoDescriptions.totalCount}`);
|
|
124
|
+
|
|
125
|
+
// Check if there are more pages and get the next page
|
|
126
|
+
if (jobsWithNoDescriptions.hasNext()) {
|
|
127
|
+
console.log('🔄 Moving to next page...');
|
|
128
|
+
jobsWithNoDescriptions = await jobsWithNoDescriptions.next();
|
|
129
|
+
pageNumber++;
|
|
130
|
+
|
|
131
|
+
// Add a delay between pages
|
|
132
|
+
console.log('Waiting 3 seconds before next page...');
|
|
133
|
+
await new Promise(resolve => setTimeout(resolve, 3000));
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
} while (jobsWithNoDescriptions.hasNext());
|
|
137
|
+
|
|
138
|
+
console.log(`\n✅ Finished updating ALL job descriptions using pagination!`);
|
|
139
|
+
console.log(`📊 Final Results:`);
|
|
140
|
+
console.log(` Total pages processed: ${pageNumber}`);
|
|
141
|
+
console.log(` Total jobs processed: ${totalProcessed}`);
|
|
142
|
+
console.log(` Total updated: ${totalUpdated}`);
|
|
143
|
+
console.log(` Total failed: ${totalFailed}`);
|
|
144
|
+
|
|
145
|
+
return {
|
|
146
|
+
success: true,
|
|
147
|
+
totalPages: pageNumber,
|
|
148
|
+
totalProcessed: totalProcessed,
|
|
149
|
+
totalUpdated: totalUpdated,
|
|
150
|
+
totalFailed: totalFailed,
|
|
151
|
+
message: `Successfully updated ${totalUpdated} job descriptions out of ${totalProcessed} total jobs across ${pageNumber} pages`
|
|
152
|
+
};
|
|
153
|
+
|
|
154
|
+
} catch (error) {
|
|
155
|
+
console.error('❌ Error in updateJobDescriptions:', error);
|
|
156
|
+
throw error;
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
export async function aggregateJobsByFieldToCMS({ field, collection }) {
|
|
161
|
+
console.log(`counting jobs per ${field}.`);
|
|
162
|
+
let jobsPerField = {};
|
|
163
|
+
let cityLocations = {};
|
|
164
|
+
let query = wixData.query("Jobs").limit(1000);
|
|
165
|
+
let results = await query.find();
|
|
166
|
+
let page = 1;
|
|
167
|
+
do {
|
|
168
|
+
console.log(`Page ${page}: ${results.items.length} jobs.`);
|
|
169
|
+
for (const job of results.items) {
|
|
170
|
+
if (!job[field])
|
|
171
|
+
{
|
|
172
|
+
throw new Error(`Job ${job._id} has no ${field} field`);
|
|
173
|
+
}
|
|
174
|
+
jobsPerField[job[field]] = (jobsPerField[job[field]] || 0) + 1;
|
|
175
|
+
if (field === 'cityText' && !cityLocations[job[field]]) {
|
|
176
|
+
cityLocations[job[field]] = job.location;
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
if (results.hasNext()) {
|
|
180
|
+
results = await results.next();
|
|
181
|
+
page++;
|
|
182
|
+
}
|
|
183
|
+
} while (results.hasNext());
|
|
184
|
+
let toSave = [];
|
|
185
|
+
if (field === 'cityText') {
|
|
186
|
+
toSave = Object.entries(jobsPerField).map(([value, amount]) => {
|
|
187
|
+
const loc = cityLocations[value] || {};
|
|
188
|
+
value = normalizeCityName(value);
|
|
189
|
+
|
|
190
|
+
return {
|
|
191
|
+
title: value,
|
|
192
|
+
_id: value.replace(/\s+/g, ''),
|
|
193
|
+
count: amount,
|
|
194
|
+
location: loc,
|
|
195
|
+
countryCode: loc.countryCode,
|
|
196
|
+
country: loc.country,
|
|
197
|
+
region: loc.region,
|
|
198
|
+
city: loc.city,
|
|
199
|
+
manual: loc.manual.toString(),
|
|
200
|
+
remote: loc.remote.toString(),
|
|
201
|
+
regionCode: loc.regionCode,
|
|
202
|
+
latitude: loc.latitude,
|
|
203
|
+
longitude: loc.longitude
|
|
204
|
+
};
|
|
205
|
+
});
|
|
206
|
+
}
|
|
207
|
+
else{
|
|
208
|
+
// Prepare array for bulkSave
|
|
209
|
+
toSave = Object.entries(jobsPerField).map(([value, amount]) => ({
|
|
210
|
+
title: value,
|
|
211
|
+
_id: value.replace(/\s+/g, ''),
|
|
212
|
+
count: amount
|
|
213
|
+
}));
|
|
214
|
+
}
|
|
215
|
+
if (toSave.length === 0) {
|
|
216
|
+
console.log('No jobs found.');
|
|
217
|
+
return { success: true, message: 'No jobs to save.' };
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
try {
|
|
221
|
+
const saveResult = await wixData.bulkSave(collection, toSave);
|
|
222
|
+
console.log(`Saved ${toSave.length} ${field} counts to ${collection}.`);
|
|
223
|
+
return { success: true, saved: toSave.length, result: saveResult };
|
|
224
|
+
} catch (err) {
|
|
225
|
+
console.error(`Error saving jobs per ${field}:`, err);
|
|
226
|
+
return { success: false, error: err.message };
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
async function getJobsWithNoDescriptions() {
|
|
231
|
+
|
|
232
|
+
let jobswithoutdescriptionsQuery = await wixData.query("Jobs").limit(1000).isEmpty("jobDescription").find(); // with 900 as the limit, 429 error won't happen
|
|
233
|
+
return jobswithoutdescriptionsQuery;
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
export async function referenceJobsToField({
|
|
237
|
+
referenceField, // e.g., "city" or "department"
|
|
238
|
+
sourceCollection, // e.g., "cities" or "departments"
|
|
239
|
+
jobField, // e.g., "cityText" or "department"
|
|
240
|
+
|
|
241
|
+
}) {
|
|
242
|
+
// Fetch all source items (cities or departments)
|
|
243
|
+
const sources = await wixData.query(sourceCollection).limit(1000).find();
|
|
244
|
+
const sourceMap = {};
|
|
245
|
+
for (const item of sources.items) {
|
|
246
|
+
sourceMap[item.title] = item._id;
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
// Fetch all jobs
|
|
250
|
+
let jobsResults = await wixData.query("Jobs").limit(1000).find();
|
|
251
|
+
let jobsToUpdate = [];
|
|
252
|
+
console.log('jobsResults',jobsResults.items);
|
|
253
|
+
|
|
254
|
+
do {
|
|
255
|
+
for (const job of jobsResults.items) {
|
|
256
|
+
const refId = sourceMap[job[jobField]];
|
|
257
|
+
if (refId) {
|
|
258
|
+
jobsToUpdate.push({
|
|
259
|
+
...job,
|
|
260
|
+
[referenceField]: refId
|
|
261
|
+
});
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
if (jobsResults.hasNext()) {
|
|
265
|
+
jobsResults = await jobsResults.next();
|
|
266
|
+
} else {
|
|
267
|
+
break;
|
|
268
|
+
}
|
|
269
|
+
} while (true);
|
|
270
|
+
|
|
271
|
+
// Remove system fields that cannot be updated
|
|
272
|
+
jobsToUpdate = jobsToUpdate.map(job => {
|
|
273
|
+
const { _createdDate, _updatedDate, ...rest } = job;
|
|
274
|
+
return rest;
|
|
275
|
+
});
|
|
276
|
+
|
|
277
|
+
// Bulk update in chunks of 1000
|
|
278
|
+
const chunkSize = 1000;
|
|
279
|
+
await chunkedBulkOperation({
|
|
280
|
+
items: jobsToUpdate,
|
|
281
|
+
chunkSize,
|
|
282
|
+
processChunk: async (chunk) => {
|
|
283
|
+
await wixData.bulkUpdate("Jobs", chunk);
|
|
284
|
+
}
|
|
285
|
+
});
|
|
286
|
+
|
|
287
|
+
return { success: true, updated: jobsToUpdate.length };
|
|
288
|
+
}
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
import { fetch } from 'wix-fetch';
|
|
2
|
+
|
|
3
|
+
async function makeSmartRecruitersRequest(path) {
|
|
4
|
+
// const baseUrl = 'https://api.smartrecruiters.com'; // PROD
|
|
5
|
+
const baseUrl = 'https://aoxley54.wixstudio.com/external-template/_functions'; // TEST
|
|
6
|
+
const fullUrl = `${baseUrl}${path}`;
|
|
7
|
+
//console.log(`Making request to: ${fullUrl}`);
|
|
8
|
+
try {
|
|
9
|
+
const response = await fetch(fullUrl, {
|
|
10
|
+
method: 'GET',
|
|
11
|
+
headers: {
|
|
12
|
+
'Accept-Language': 'en',
|
|
13
|
+
'accept': 'application/json',
|
|
14
|
+
'x-smarttoken': 'DCRA1-1d30ea5fe9be42d9b9ae94ff933ebef5',
|
|
15
|
+
'Cookie': 'AWSALB=GYltFw3fLKortMxHR5vIOT1CuUROyhWNIX/qL8ZnPl1/8mhOcnIsBKYslzmNJPEzSy/jvNbO+6tXpH8yqcpQJagYt57MhbKlLqTSzoNq1G/w7TjOxPGR3UTdXW0d; AWSALBCORS=GYltFw3fLKortMxHR5vIOT1CuUROyhWNIX/qL8ZnPl1/8mhOcnIsBKYslzmNJPEzSy/jvNbO+6tXpH8yqcpQJagYt57MhbKlLqTSzoNq1G/w7TjOxPGR3UTdXW0d'
|
|
16
|
+
}
|
|
17
|
+
});
|
|
18
|
+
|
|
19
|
+
if (response.ok) {
|
|
20
|
+
const data = await response.json();
|
|
21
|
+
return data;
|
|
22
|
+
} else {
|
|
23
|
+
throw new Error(`HTTP error! status: ${response.status}`);
|
|
24
|
+
}
|
|
25
|
+
} catch (error) {
|
|
26
|
+
console.error('Error making SmartRecruiters API request:', error);
|
|
27
|
+
throw error;
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
export async function fetchPositionsFromSRAPI() {
|
|
32
|
+
let allPositions = [];
|
|
33
|
+
let totalFound = 0;
|
|
34
|
+
let nextPageId = null; // Start with no page ID for the first request
|
|
35
|
+
let pageCount = 0;
|
|
36
|
+
const MAX_PAGES = 30 // Safety limit to prevent infinite loops
|
|
37
|
+
|
|
38
|
+
console.log('Starting to fetch all positions with pagination...');
|
|
39
|
+
|
|
40
|
+
do {
|
|
41
|
+
try {
|
|
42
|
+
pageCount++;
|
|
43
|
+
|
|
44
|
+
// Build the API path - first request has no page parameter, subsequent use nextPageId
|
|
45
|
+
let apiPath = '/jobs?limit=50';
|
|
46
|
+
if (nextPageId) {
|
|
47
|
+
apiPath += `&nextPageId=${nextPageId}`;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
console.log(`Fetching page ${pageCount} with path: ${apiPath}`);
|
|
51
|
+
const response = await makeSmartRecruitersRequest(apiPath);
|
|
52
|
+
|
|
53
|
+
// Add positions from this page to our collection
|
|
54
|
+
if (response.content && Array.isArray(response.content)) {
|
|
55
|
+
allPositions = allPositions.concat(response.content);
|
|
56
|
+
console.log(`Page ${pageCount}: Found ${response.content.length} positions`);
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
// Update total count from first response
|
|
60
|
+
if (pageCount === 1) {
|
|
61
|
+
totalFound = response.totalFound || 0;
|
|
62
|
+
console.log(`Total positions available: ${totalFound}`);
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
// Get the nextPageId for the next iteration
|
|
66
|
+
nextPageId = response.nextPageId && response.nextPageId !== "" ? response.nextPageId : null;
|
|
67
|
+
|
|
68
|
+
if (nextPageId) {
|
|
69
|
+
console.log(`Next page ID: ${nextPageId}`);
|
|
70
|
+
} else {
|
|
71
|
+
console.log('No more pages to fetch');
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
} catch (error) {
|
|
75
|
+
console.error(`Error fetching page ${pageCount}:`, error);
|
|
76
|
+
throw error;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
// Safety check to prevent infinite loops
|
|
80
|
+
if (pageCount >= MAX_PAGES) {
|
|
81
|
+
console.warn(`Reached maximum page limit of ${MAX_PAGES}. Stopping pagination.`);
|
|
82
|
+
break;
|
|
83
|
+
}
|
|
84
|
+
} while (nextPageId); // Continue while there's a nextPageId
|
|
85
|
+
|
|
86
|
+
console.log(`Finished fetching all pages. Total positions collected: ${allPositions.length}`);
|
|
87
|
+
|
|
88
|
+
// Return response in the same format as before, but with all positions
|
|
89
|
+
const result = {
|
|
90
|
+
totalFound: totalFound,
|
|
91
|
+
offset: 0,
|
|
92
|
+
limit: allPositions.length,
|
|
93
|
+
nextPageId: "", // Always empty since we've fetched everything
|
|
94
|
+
content: allPositions
|
|
95
|
+
};
|
|
96
|
+
|
|
97
|
+
const amountOfUniqueJobs = new Set(allPositions.map(job => job.id)).size;
|
|
98
|
+
console.log('amountOfUniqueJobs ===');
|
|
99
|
+
console.log(amountOfUniqueJobs);
|
|
100
|
+
console.log('amountOfUniqueJobs ===');
|
|
101
|
+
console.log('result ===');
|
|
102
|
+
console.log(result);
|
|
103
|
+
console.log('result ===');
|
|
104
|
+
return result;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
export async function fetchJobDescription(jobId) {
|
|
108
|
+
return await makeSmartRecruitersRequest(`/jobs/${jobId}`);
|
|
109
|
+
}
|
package/backend/index.js
CHANGED
package/backend/queries.js
CHANGED
package/backend/utils.js
ADDED
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "sr-npm",
|
|
3
|
-
"version": "1.0.
|
|
3
|
+
"version": "1.0.7",
|
|
4
4
|
"description": "",
|
|
5
5
|
"main": "index.js",
|
|
6
6
|
"scripts": {
|
|
@@ -17,7 +17,6 @@
|
|
|
17
17
|
},
|
|
18
18
|
"homepage": "https://github.com/psdevteamenterprise/sr-npm#readme",
|
|
19
19
|
"dependencies": {
|
|
20
|
-
"@
|
|
21
|
-
"@wix/essentials": "^0.1.24"
|
|
20
|
+
"@hisense-staging/velo-npm": "^3.0.10"
|
|
22
21
|
}
|
|
23
22
|
}
|
package/public/index.js
CHANGED
|
@@ -1,17 +0,0 @@
|
|
|
1
|
-
const { items } = require('@wix/data');
|
|
2
|
-
const { auth } = require('@wix/essentials');
|
|
3
|
-
|
|
4
|
-
// @wix/data does not support suppressAuth currently, so we need to elevate it
|
|
5
|
-
const wixData = {
|
|
6
|
-
insert: auth.elevate(items.insert),
|
|
7
|
-
update: auth.elevate(items.update),
|
|
8
|
-
bulkInsert: auth.elevate(items.bulkInsert),
|
|
9
|
-
query: auth.elevate(items.query),
|
|
10
|
-
save: auth.elevate(items.save),
|
|
11
|
-
remove: auth.elevate(items.remove),
|
|
12
|
-
get: auth.elevate(items.get),
|
|
13
|
-
//TODO: add other methods here as needed
|
|
14
|
-
};
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
module.exports = { wixData };
|