sr-npm 1.7.65 → 1.7.66
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.prettierrc +16 -0
- package/backend/collectionConsts.js +35 -9
- package/backend/consts.js +125 -93
- package/backend/data.js +240 -288
- package/backend/fetchPositionsFromSRAPI.js +21 -22
- package/backend/index.js +5 -6
- package/backend/queries.js +14 -11
- package/backend/utils.js +64 -5
- package/index.js +4 -5
- package/package.json +6 -3
- package/pages/index.js +4 -0
- package/pages/positionPage.js +34 -0
- package/public/index.js +4 -0
- package/public/utils.js +28 -0
package/.prettierrc
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
{
|
|
2
|
+
"singleQuote": true,
|
|
3
|
+
"trailingComma": "es5",
|
|
4
|
+
"printWidth": 100,
|
|
5
|
+
"tabWidth": 2,
|
|
6
|
+
"semi": true,
|
|
7
|
+
"bracketSpacing": true,
|
|
8
|
+
"arrowParens": "avoid",
|
|
9
|
+
"endOfLine": "lf",
|
|
10
|
+
"quoteProps": "as-needed",
|
|
11
|
+
"jsxSingleQuote": false,
|
|
12
|
+
"jsxBracketSameLine": false,
|
|
13
|
+
"requirePragma": false,
|
|
14
|
+
"insertPragma": false,
|
|
15
|
+
"proseWrap": "preserve"
|
|
16
|
+
}
|
|
@@ -1,10 +1,35 @@
|
|
|
1
1
|
const COLLECTIONS = {
|
|
2
|
-
AMOUNT_OF_JOBS_PER_DEPARTMENT: '
|
|
3
|
-
CITIES: '
|
|
4
|
-
JOBS: '
|
|
2
|
+
AMOUNT_OF_JOBS_PER_DEPARTMENT: 'AmountOfJobsPerDepartment1',
|
|
3
|
+
CITIES: 'cities1',
|
|
4
|
+
JOBS: 'Jobs1',
|
|
5
5
|
API_KEY: 'ApiKey',
|
|
6
6
|
}
|
|
7
|
-
|
|
7
|
+
const JOBS_COLLECTION_FIELDS = {
|
|
8
|
+
LOCATION: 'location',
|
|
9
|
+
TITLE: 'title',
|
|
10
|
+
LOCATION_ADDRESS: 'locationAddress',
|
|
11
|
+
POSTING_STATUS: 'postingStatus',
|
|
12
|
+
COUNTRY: 'country',
|
|
13
|
+
DEPARTMENT: 'department',
|
|
14
|
+
LANGUAGE: 'language',
|
|
15
|
+
REMOTE: 'remote',
|
|
16
|
+
JOB_DESCRIPTION: 'jobDescription',
|
|
17
|
+
DEPARTMENT_REF: 'departmentRef',
|
|
18
|
+
CITY: 'city',
|
|
19
|
+
CITY_TEXT: 'cityText',
|
|
20
|
+
}
|
|
21
|
+
const AMOUNT_OF_JOBS_PER_DEPARTMENT_COLLECTION_FIELDS = {
|
|
22
|
+
TITLE: 'title',
|
|
23
|
+
COUNT: 'count',
|
|
24
|
+
IMAGE: 'image',
|
|
25
|
+
}
|
|
26
|
+
const CITIES_COLLECTION_FIELDS = {
|
|
27
|
+
TITLE: 'title',
|
|
28
|
+
CITY: 'city',
|
|
29
|
+
LOCATION_ADDRESS: 'locationAddress',
|
|
30
|
+
COUNT: 'count',
|
|
31
|
+
COUNTRY: 'country',
|
|
32
|
+
}
|
|
8
33
|
const COLLECTIONS_FIELDS = {
|
|
9
34
|
AMOUNT_OF_JOBS_PER_DEPARTMENT: [
|
|
10
35
|
{key:'title', type: 'TEXT'},
|
|
@@ -28,10 +53,9 @@ const COLLECTIONS_FIELDS = {
|
|
|
28
53
|
{key:'language', type: 'TEXT'},
|
|
29
54
|
{key:'remote', type: 'BOOLEAN'},
|
|
30
55
|
{key:'jobDescription', type: 'OBJECT'},
|
|
31
|
-
{key:'cityText', type: 'TEXT'},
|
|
32
|
-
{key:'
|
|
33
|
-
{key:'
|
|
34
|
-
{key:'city', type: 'REFERENCE', typeMetadata: { reference: { referencedCollectionId: 'cities' } } },
|
|
56
|
+
{key:'cityText', type: 'TEXT'},
|
|
57
|
+
{key:'departmentRef', type: 'REFERENCE', typeMetadata: { reference: { referencedCollectionId: 'AmountOfJobsPerDepartment1' } } },
|
|
58
|
+
{key:'city', type: 'REFERENCE', typeMetadata: { reference: { referencedCollectionId: 'cities1' } } },
|
|
35
59
|
],
|
|
36
60
|
API_KEY: [
|
|
37
61
|
{key:'token', type: 'TEXT'},
|
|
@@ -39,8 +63,10 @@ const COLLECTIONS_FIELDS = {
|
|
|
39
63
|
};
|
|
40
64
|
|
|
41
65
|
|
|
42
|
-
|
|
43
66
|
module.exports = {
|
|
44
67
|
COLLECTIONS,
|
|
45
68
|
COLLECTIONS_FIELDS,
|
|
69
|
+
JOBS_COLLECTION_FIELDS,
|
|
70
|
+
AMOUNT_OF_JOBS_PER_DEPARTMENT_COLLECTION_FIELDS,
|
|
71
|
+
CITIES_COLLECTION_FIELDS,
|
|
46
72
|
};
|
package/backend/consts.js
CHANGED
|
@@ -1,10 +1,19 @@
|
|
|
1
|
-
const {
|
|
1
|
+
const {
|
|
2
|
+
saveJobsDataToCMS,
|
|
3
|
+
saveJobsDescriptionsAndLocationToCMS,
|
|
4
|
+
aggregateJobsByFieldToCMS,
|
|
5
|
+
referenceJobsToField,
|
|
6
|
+
createApiKeyCollectionAndFillIt,
|
|
7
|
+
} = require('./data');
|
|
2
8
|
const { createCollectionIfMissing } = require('@hisense-staging/velo-npm/backend');
|
|
3
|
-
const { COLLECTIONS, COLLECTIONS_FIELDS } = require('./collectionConsts');
|
|
9
|
+
const { COLLECTIONS, COLLECTIONS_FIELDS, JOBS_COLLECTION_FIELDS } = require('./collectionConsts');
|
|
10
|
+
|
|
11
|
+
const QUERY_MAX_LIMIT = 1000;
|
|
12
|
+
|
|
4
13
|
const TASKS_NAMES = {
|
|
5
14
|
SYNC_JOBS: 'syncJobsFromSRAPIToCMS',
|
|
6
15
|
INSERT_JOBS_TO_CMS: 'insertJobsToCMS',
|
|
7
|
-
|
|
16
|
+
INSERT_JOBS_DESCRIPTIONS_TO_CMS: 'insertJobsDescriptionsToCMS',
|
|
8
17
|
FILL_JOBS_PER_CITY_COLLECTION: 'fillJobsPerCityCollection',
|
|
9
18
|
FILL_JOBS_PER_DEPARTMENT_COLLECTION: 'fillJobsPerDepartmentCollection',
|
|
10
19
|
REFERENCE_JOBS_TO_LOCATIONS: 'referenceJobsToLocations',
|
|
@@ -17,103 +26,126 @@ const TASKS_NAMES = {
|
|
|
17
26
|
|
|
18
27
|
|
|
19
28
|
const TASKS = {
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
}
|
|
106
|
-
|
|
29
|
+
[TASKS_NAMES.SYNC_JOBS]: {
|
|
30
|
+
name: TASKS_NAMES.SYNC_JOBS,
|
|
31
|
+
childTasks: [
|
|
32
|
+
{ name: TASKS_NAMES.CREATE_JOBS_COLLECTION },
|
|
33
|
+
{ name: TASKS_NAMES.CREATE_CITIES_COLLECTION },
|
|
34
|
+
{ name: TASKS_NAMES.CREATE_AMOUNT_OF_JOBS_PER_DEPARTMENT_COLLECTION },
|
|
35
|
+
{ name: TASKS_NAMES.INSERT_JOBS_TO_CMS },
|
|
36
|
+
{ name: TASKS_NAMES.INSERT_JOBS_DESCRIPTIONS_TO_CMS },
|
|
37
|
+
{ name: TASKS_NAMES.FILL_JOBS_PER_CITY_COLLECTION },
|
|
38
|
+
{ name: TASKS_NAMES.FILL_JOBS_PER_DEPARTMENT_COLLECTION },
|
|
39
|
+
{ name: TASKS_NAMES.REFERENCE_JOBS_TO_LOCATIONS },
|
|
40
|
+
{ name: TASKS_NAMES.REFERENCE_JOBS_TO_DEPARTMENT },
|
|
41
|
+
],
|
|
42
|
+
scheduleChildrenSequentially: true,
|
|
43
|
+
estimatedDurationSec: 30,
|
|
44
|
+
},
|
|
45
|
+
[TASKS_NAMES.CREATE_JOBS_COLLECTION]: {
|
|
46
|
+
name: TASKS_NAMES.CREATE_JOBS_COLLECTION,
|
|
47
|
+
getIdentifier: () => 'SHOULD_NEVER_SKIP',
|
|
48
|
+
process: () => createCollectionIfMissing(COLLECTIONS.JOBS, COLLECTIONS_FIELDS.JOBS,{ insert: 'ADMIN', update: 'ADMIN', remove: 'ADMIN', read: 'ANYONE' }),
|
|
49
|
+
shouldSkipCheck: () => false,
|
|
50
|
+
estimatedDurationSec: 3,
|
|
51
|
+
},
|
|
52
|
+
[TASKS_NAMES.CREATE_CITIES_COLLECTION]: {
|
|
53
|
+
name: TASKS_NAMES.CREATE_CITIES_COLLECTION,
|
|
54
|
+
getIdentifier: () => 'SHOULD_NEVER_SKIP',
|
|
55
|
+
process: () => createCollectionIfMissing(COLLECTIONS.CITIES, COLLECTIONS_FIELDS.CITIES),
|
|
56
|
+
shouldSkipCheck: () => false,
|
|
57
|
+
estimatedDurationSec: 3,
|
|
58
|
+
},
|
|
59
|
+
[TASKS_NAMES.CREATE_AMOUNT_OF_JOBS_PER_DEPARTMENT_COLLECTION]: {
|
|
60
|
+
name: TASKS_NAMES.CREATE_AMOUNT_OF_JOBS_PER_DEPARTMENT_COLLECTION,
|
|
61
|
+
getIdentifier: () => 'SHOULD_NEVER_SKIP',
|
|
62
|
+
process: () =>
|
|
63
|
+
createCollectionIfMissing(
|
|
64
|
+
COLLECTIONS.AMOUNT_OF_JOBS_PER_DEPARTMENT,
|
|
65
|
+
COLLECTIONS_FIELDS.AMOUNT_OF_JOBS_PER_DEPARTMENT
|
|
66
|
+
),
|
|
67
|
+
shouldSkipCheck: () => false,
|
|
68
|
+
estimatedDurationSec: 3,
|
|
69
|
+
},
|
|
70
|
+
[TASKS_NAMES.INSERT_JOBS_TO_CMS]: {
|
|
71
|
+
name: TASKS_NAMES.INSERT_JOBS_TO_CMS,
|
|
72
|
+
getIdentifier: () => 'SHOULD_NEVER_SKIP',
|
|
73
|
+
process: saveJobsDataToCMS,
|
|
74
|
+
shouldSkipCheck: () => false,
|
|
75
|
+
estimatedDurationSec: 20,
|
|
76
|
+
},
|
|
77
|
+
[TASKS_NAMES.INSERT_JOBS_DESCRIPTIONS_TO_CMS]: {
|
|
78
|
+
name: TASKS_NAMES.INSERT_JOBS_DESCRIPTIONS_TO_CMS,
|
|
79
|
+
getIdentifier: () => 'SHOULD_NEVER_SKIP',
|
|
80
|
+
process: saveJobsDescriptionsAndLocationToCMS,
|
|
81
|
+
shouldSkipCheck: () => false,
|
|
82
|
+
estimatedDurationSec: 20,
|
|
83
|
+
},
|
|
84
|
+
[TASKS_NAMES.FILL_JOBS_PER_CITY_COLLECTION]: {
|
|
85
|
+
name: TASKS_NAMES.FILL_JOBS_PER_CITY_COLLECTION,
|
|
86
|
+
getIdentifier: () => 'SHOULD_NEVER_SKIP',
|
|
87
|
+
process: () =>
|
|
88
|
+
aggregateJobsByFieldToCMS({
|
|
89
|
+
field: JOBS_COLLECTION_FIELDS.CITY_TEXT,
|
|
90
|
+
collection: COLLECTIONS.CITIES,
|
|
91
|
+
}),
|
|
92
|
+
shouldSkipCheck: () => false,
|
|
93
|
+
estimatedDurationSec: 3,
|
|
94
|
+
},
|
|
95
|
+
[TASKS_NAMES.FILL_JOBS_PER_DEPARTMENT_COLLECTION]: {
|
|
96
|
+
name: TASKS_NAMES.FILL_JOBS_PER_DEPARTMENT_COLLECTION,
|
|
97
|
+
getIdentifier: () => 'SHOULD_NEVER_SKIP',
|
|
98
|
+
process: () =>
|
|
99
|
+
aggregateJobsByFieldToCMS({
|
|
100
|
+
field: JOBS_COLLECTION_FIELDS.DEPARTMENT,
|
|
101
|
+
collection: COLLECTIONS.AMOUNT_OF_JOBS_PER_DEPARTMENT,
|
|
102
|
+
}),
|
|
103
|
+
shouldSkipCheck: () => false,
|
|
104
|
+
estimatedDurationSec: 3,
|
|
105
|
+
},
|
|
106
|
+
[TASKS_NAMES.REFERENCE_JOBS_TO_LOCATIONS]: {
|
|
107
|
+
name: TASKS_NAMES.REFERENCE_JOBS_TO_LOCATIONS,
|
|
108
|
+
getIdentifier: () => 'SHOULD_NEVER_SKIP',
|
|
109
|
+
process: () =>
|
|
110
|
+
referenceJobsToField({
|
|
111
|
+
referenceField: JOBS_COLLECTION_FIELDS.CITY,
|
|
112
|
+
sourceCollection: COLLECTIONS.CITIES,
|
|
113
|
+
jobField: JOBS_COLLECTION_FIELDS.CITY_TEXT,
|
|
114
|
+
}),
|
|
115
|
+
shouldSkipCheck: () => false,
|
|
116
|
+
estimatedDurationSec: 3,
|
|
117
|
+
},
|
|
118
|
+
[TASKS_NAMES.REFERENCE_JOBS_TO_DEPARTMENT]: {
|
|
119
|
+
name: TASKS_NAMES.REFERENCE_JOBS_TO_DEPARTMENT,
|
|
120
|
+
getIdentifier: () => 'SHOULD_NEVER_SKIP',
|
|
121
|
+
process: () =>
|
|
122
|
+
referenceJobsToField({
|
|
123
|
+
referenceField: JOBS_COLLECTION_FIELDS.DEPARTMENT_REF,
|
|
124
|
+
sourceCollection: COLLECTIONS.AMOUNT_OF_JOBS_PER_DEPARTMENT,
|
|
125
|
+
jobField: JOBS_COLLECTION_FIELDS.DEPARTMENT,
|
|
126
|
+
}),
|
|
127
|
+
shouldSkipCheck: () => false,
|
|
128
|
+
estimatedDurationSec: 3,
|
|
129
|
+
},
|
|
130
|
+
[TASKS_NAMES.CREATE_API_KEY_COLLECTION_AND_FILL_IT]: {
|
|
131
|
+
name: TASKS_NAMES.CREATE_API_KEY_COLLECTION_AND_FILL_IT,
|
|
132
|
+
getIdentifier:()=> "SHOULD_NEVER_SKIP",
|
|
133
|
+
process:()=>createApiKeyCollectionAndFillIt(),
|
|
134
|
+
shouldSkipCheck:()=>false,
|
|
135
|
+
estimatedDurationSec:3
|
|
136
|
+
}
|
|
137
|
+
};
|
|
107
138
|
|
|
108
139
|
|
|
109
140
|
|
|
110
141
|
const TASK_TYPE = {
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
142
|
+
SCHEDULED: 'scheduled',
|
|
143
|
+
EVENT: 'event',
|
|
144
|
+
};
|
|
114
145
|
|
|
115
146
|
module.exports = {
|
|
116
147
|
TASKS_NAMES,
|
|
117
148
|
TASK_TYPE,
|
|
118
149
|
TASKS,
|
|
150
|
+
QUERY_MAX_LIMIT
|
|
119
151
|
};
|
package/backend/data.js
CHANGED
|
@@ -1,321 +1,272 @@
|
|
|
1
1
|
const { items: wixData } = require('@wix/data');
|
|
2
2
|
const { fetchPositionsFromSRAPI, fetchJobDescription } = require('./fetchPositionsFromSRAPI');
|
|
3
|
-
const { chunkedBulkOperation } = require('./utils');
|
|
4
3
|
const { createCollectionIfMissing } = require('@hisense-staging/velo-npm/backend');
|
|
5
4
|
const { COLLECTIONS, COLLECTIONS_FIELDS } = require('./collectionConsts');
|
|
6
5
|
const { secrets } = require("@wix/secrets");
|
|
7
6
|
const { auth } = require('@wix/essentials');
|
|
7
|
+
const { chunkedBulkOperation, delay, countJobsPerGivenField, fillCityLocationAndLocationAddress ,prepareToSaveArray,normalizeCityName} = require('./utils');
|
|
8
|
+
const { getAllPositions } = require('./queries');
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
function validatePosition(position) {
|
|
12
|
+
if (!position.id) {
|
|
13
|
+
throw new Error('Position id is required');
|
|
14
|
+
}
|
|
15
|
+
if (!position.title) {
|
|
16
|
+
throw new Error('Position title is required');
|
|
17
|
+
}
|
|
18
|
+
if (!position.department || !position.department.label) {
|
|
19
|
+
throw new Error('Position department is required and must have a label');
|
|
20
|
+
}
|
|
21
|
+
if (!position.location || !position.location.city || !position.location.remote) {
|
|
22
|
+
throw new Error('Position location is required and must have a city and remote');
|
|
23
|
+
}
|
|
8
24
|
|
|
9
|
-
// Utility function to normalize city names
|
|
10
|
-
function normalizeCityName(city) {
|
|
11
|
-
if (!city) return city;
|
|
12
|
-
// Remove accents/diacritics, trim whitespace
|
|
13
|
-
return city.normalize('NFD').replace(/\p{Diacritic}/gu, '').trim();
|
|
14
25
|
}
|
|
15
26
|
|
|
16
|
-
async function
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
27
|
+
async function saveJobsDataToCMS() {
|
|
28
|
+
const positions = await fetchPositionsFromSRAPI();
|
|
29
|
+
// bulk insert to jobs collection without descriptions first
|
|
30
|
+
const jobsData = positions.content.map(position => {
|
|
31
|
+
validatePosition(position);
|
|
32
|
+
const basicJob = {
|
|
33
|
+
_id: position.id,
|
|
34
|
+
title: position.title,
|
|
35
|
+
department: position.department.label,
|
|
36
|
+
cityText: normalizeCityName(position.location.city),
|
|
37
|
+
location: position.location,
|
|
38
|
+
country: position.location.country,
|
|
39
|
+
remote: position.location.remote,
|
|
40
|
+
language: position.language.label,
|
|
41
|
+
postingStatus: position.postingStatus,
|
|
42
|
+
jobDescription: null, // Will be filled later
|
|
43
|
+
};
|
|
44
|
+
return basicJob;
|
|
45
|
+
});
|
|
46
|
+
|
|
47
|
+
const chunkSize = 1000;
|
|
48
|
+
let totalSaved = 0;
|
|
49
|
+
const totalChunks = Math.ceil(jobsData.length / chunkSize);
|
|
50
|
+
|
|
51
|
+
console.log(
|
|
52
|
+
`Processing ${jobsData.length} jobs in ${totalChunks} chunks of max ${chunkSize} items each`
|
|
53
|
+
);
|
|
54
|
+
|
|
55
|
+
await chunkedBulkOperation({
|
|
56
|
+
items: jobsData,
|
|
57
|
+
chunkSize,
|
|
58
|
+
processChunk: async (chunk, chunkNumber) => {
|
|
59
|
+
console.log(`Saving chunk ${chunkNumber}/${totalChunks}: ${chunk.length} jobs`);
|
|
60
|
+
try {
|
|
61
|
+
const result = await wixData.bulkSave('Jobs1', chunk);
|
|
62
|
+
const saved = result.inserted + result.updated || chunk.length;
|
|
63
|
+
totalSaved += saved;
|
|
64
|
+
console.log(
|
|
65
|
+
`✓ Chunk ${chunkNumber} saved successfully. Inserted: ${result.inserted}, Updated: ${result.updated}`
|
|
66
|
+
);
|
|
67
|
+
} catch (error) {
|
|
68
|
+
console.error(`✗ Error saving chunk ${chunkNumber}:`, error);
|
|
69
|
+
throw error;
|
|
70
|
+
}
|
|
71
|
+
},
|
|
72
|
+
});
|
|
73
|
+
|
|
74
|
+
console.log(`✓ All chunks processed. Total jobs saved: ${totalSaved}/${jobsData.length}`);
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
async function saveJobsDescriptionsAndLocationToCMS() {
|
|
78
|
+
console.log('🚀 Starting job descriptions update process for ALL jobs');
|
|
79
|
+
|
|
80
|
+
try {
|
|
81
|
+
let jobsWithNoDescriptions = await getJobsWithNoDescriptions();
|
|
82
|
+
let totalUpdated = 0;
|
|
83
|
+
let totalFailed = 0;
|
|
84
|
+
let totalProcessed = 0;
|
|
85
|
+
|
|
86
|
+
console.log(
|
|
87
|
+
`Total jobs in database without descriptions: ${jobsWithNoDescriptions?.items?.length}`
|
|
88
|
+
);
|
|
89
|
+
|
|
90
|
+
if (jobsWithNoDescriptions.items.length === 0) {
|
|
91
|
+
console.log('No jobs found in database');
|
|
92
|
+
return { success: true, message: 'No jobs found' };
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
const API_CHUNK_SIZE = 80;
|
|
97
|
+
const pageChunks = Math.ceil(jobsWithNoDescriptions.items.length / API_CHUNK_SIZE);
|
|
40
98
|
|
|
41
99
|
await chunkedBulkOperation({
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
100
|
+
items: jobsWithNoDescriptions.items,
|
|
101
|
+
chunkSize: API_CHUNK_SIZE,
|
|
102
|
+
processChunk: async (chunk, chunkNumber) => {
|
|
103
|
+
console.log(` Processing API chunk ${chunkNumber}/${pageChunks} (${chunk.length} jobs)`);
|
|
104
|
+
const chunkPromises = chunk.map(async job => {
|
|
105
|
+
try {
|
|
106
|
+
const jobDetails = await fetchJobDescription(job._id);
|
|
107
|
+
const jobLocation = fetchJobLocation(jobDetails);
|
|
108
|
+
|
|
109
|
+
const updatedJob = {
|
|
110
|
+
...job,
|
|
111
|
+
locationAddress: jobLocation,
|
|
112
|
+
jobDescription: jobDetails.jobAd.sections,
|
|
113
|
+
};
|
|
114
|
+
await wixData.update('Jobs1', updatedJob);
|
|
115
|
+
return { success: true, jobId: job._id, title: job.title };
|
|
116
|
+
} catch (error) {
|
|
117
|
+
console.error(` ❌ Failed to update ${job.title} (${job._id}):`, error);
|
|
118
|
+
return { success: false, jobId: job._id, title: job.title, error: error.message };
|
|
119
|
+
}
|
|
120
|
+
});
|
|
121
|
+
const chunkResults = await Promise.all(chunkPromises);
|
|
122
|
+
const chunkSuccesses = chunkResults.filter(r => r.success).length;
|
|
123
|
+
const chunkFailures = chunkResults.filter(r => !r.success).length;
|
|
124
|
+
totalUpdated += chunkSuccesses;
|
|
125
|
+
totalFailed += chunkFailures;
|
|
126
|
+
totalProcessed += chunk.length;
|
|
127
|
+
console.log(
|
|
128
|
+
` API chunk ${chunkNumber} completed: ${chunkSuccesses} success, ${chunkFailures} failed`
|
|
129
|
+
);
|
|
130
|
+
if (chunkNumber * API_CHUNK_SIZE < jobsWithNoDescriptions.items.length) {
|
|
131
|
+
console.log(' Waiting 2 seconds before next API chunk...');
|
|
132
|
+
await delay(2000);
|
|
55
133
|
}
|
|
134
|
+
},
|
|
56
135
|
});
|
|
57
|
-
|
|
58
|
-
console.log(`✓ All chunks processed. Total jobs saved: ${totalSaved}/${jobsData.length}`);
|
|
59
136
|
|
|
60
|
-
}
|
|
61
137
|
|
|
62
|
-
async function saveJobsDescriptionsAndLocationApplyUrlToCMS() {
|
|
63
|
-
|
|
64
|
-
console.log('🚀 Starting job descriptions update process for ALL jobs using pagination...');
|
|
65
|
-
|
|
66
|
-
try {
|
|
67
|
-
let jobsWithNoDescriptions = await getJobsWithNoDescriptions();
|
|
68
|
-
let totalUpdated = 0;
|
|
69
|
-
let totalFailed = 0;
|
|
70
|
-
let totalProcessed = 0;
|
|
71
|
-
let pageNumber = 1;
|
|
72
|
-
|
|
73
|
-
// Start with the first page query - limit to 100 jobs per page
|
|
74
|
-
// let jobsQuery = await wixData.query("Jobs").limit(300).find();
|
|
75
|
-
|
|
76
|
-
console.log(`Total jobs in database without descriptions: ${jobsWithNoDescriptions.totalCount}`);
|
|
77
|
-
|
|
78
|
-
if (jobsWithNoDescriptions.totalCount === 0) {
|
|
79
|
-
console.log('No jobs found in database');
|
|
80
|
-
return { success: true, message: 'No jobs found' };
|
|
81
|
-
}
|
|
82
138
|
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
const jobDetails = await fetchJobDescription(job._id);
|
|
101
|
-
const jobLocation = fetchJobLocation(jobDetails)
|
|
102
|
-
const applyLink = fetchApplyLink(jobDetails);
|
|
103
|
-
|
|
104
|
-
const updatedJob = {
|
|
105
|
-
...job,
|
|
106
|
-
locationAddress: jobLocation,
|
|
107
|
-
jobDescription: jobDetails.jobAd.sections,
|
|
108
|
-
applyLink: applyLink
|
|
109
|
-
};
|
|
110
|
-
await wixData.update(COLLECTIONS.JOBS, updatedJob);
|
|
111
|
-
// console.log(` ✅ Updated description for: ${job.title}`);
|
|
112
|
-
return { success: true, jobId: job._id, title: job.title };
|
|
113
|
-
} catch (error) {
|
|
114
|
-
console.error(` ❌ Failed to update ${job.title} (${job._id}):`, error);
|
|
115
|
-
return { success: false, jobId: job._id, title: job.title, error: error.message };
|
|
116
|
-
}
|
|
117
|
-
});
|
|
118
|
-
const chunkResults = await Promise.all(chunkPromises);
|
|
119
|
-
const chunkSuccesses = chunkResults.filter(r => r.success).length;
|
|
120
|
-
const chunkFailures = chunkResults.filter(r => !r.success).length;
|
|
121
|
-
totalUpdated += chunkSuccesses;
|
|
122
|
-
totalFailed += chunkFailures;
|
|
123
|
-
totalProcessed += chunk.length;
|
|
124
|
-
console.log(` API chunk ${chunkNumber} completed: ${chunkSuccesses} success, ${chunkFailures} failed`);
|
|
125
|
-
if (chunkNumber * API_CHUNK_SIZE < currentPageJobs.length) {
|
|
126
|
-
console.log(' Waiting 2 seconds before next API chunk...');
|
|
127
|
-
await new Promise(resolve => setTimeout(resolve, 2000));
|
|
128
|
-
}
|
|
129
|
-
}
|
|
130
|
-
});
|
|
131
|
-
|
|
132
|
-
console.log(`📄 Page ${pageNumber} completed. Updated: ${totalUpdated}, Failed: ${totalFailed}, Total processed: ${totalProcessed}/${jobsWithNoDescriptions.totalCount}`);
|
|
133
|
-
|
|
134
|
-
// Check if there are more pages and get the next page
|
|
135
|
-
if (jobsWithNoDescriptions.hasNext()) {
|
|
136
|
-
console.log('🔄 Moving to next page...');
|
|
137
|
-
jobsWithNoDescriptions = await jobsWithNoDescriptions.next();
|
|
138
|
-
pageNumber++;
|
|
139
|
-
|
|
140
|
-
// Add a delay between pages
|
|
141
|
-
console.log('Waiting 3 seconds before next page...');
|
|
142
|
-
await new Promise(resolve => setTimeout(resolve, 3000));
|
|
143
|
-
}
|
|
144
|
-
|
|
145
|
-
} while (jobsWithNoDescriptions.hasNext());
|
|
146
|
-
|
|
147
|
-
console.log(`\n✅ Finished updating ALL job descriptions using pagination!`);
|
|
148
|
-
console.log(`📊 Final Results:`);
|
|
149
|
-
console.log(` Total pages processed: ${pageNumber}`);
|
|
150
|
-
console.log(` Total jobs processed: ${totalProcessed}`);
|
|
151
|
-
console.log(` Total updated: ${totalUpdated}`);
|
|
152
|
-
console.log(` Total failed: ${totalFailed}`);
|
|
153
|
-
|
|
154
|
-
return {
|
|
155
|
-
success: true,
|
|
156
|
-
totalPages: pageNumber,
|
|
157
|
-
totalProcessed: totalProcessed,
|
|
158
|
-
totalUpdated: totalUpdated,
|
|
159
|
-
totalFailed: totalFailed,
|
|
160
|
-
message: `Successfully updated ${totalUpdated} job descriptions out of ${totalProcessed} total jobs across ${pageNumber} pages`
|
|
161
|
-
};
|
|
162
|
-
|
|
163
|
-
} catch (error) {
|
|
164
|
-
console.error('❌ Error in updateJobDescriptions:', error);
|
|
165
|
-
throw error;
|
|
166
|
-
}
|
|
139
|
+
console.log(`\n✅ Finished updating ALL job descriptions`);
|
|
140
|
+
console.log(`📊 Final Results:`);
|
|
141
|
+
console.log(` Total jobs processed: ${totalProcessed}`);
|
|
142
|
+
console.log(` Total updated: ${totalUpdated}`);
|
|
143
|
+
console.log(` Total failed: ${totalFailed}`);
|
|
144
|
+
|
|
145
|
+
return {
|
|
146
|
+
success: true,
|
|
147
|
+
totalProcessed: totalProcessed,
|
|
148
|
+
totalUpdated: totalUpdated,
|
|
149
|
+
totalFailed: totalFailed,
|
|
150
|
+
message: `Successfully updated ${totalUpdated} job descriptions out of ${totalProcessed} total jobs`,
|
|
151
|
+
};
|
|
152
|
+
} catch (error) {
|
|
153
|
+
console.error('❌ Error in updateJobDescriptions:', error);
|
|
154
|
+
throw error;
|
|
155
|
+
}
|
|
167
156
|
}
|
|
168
157
|
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
const cityLocationAddress={}
|
|
176
|
-
let page = 1;
|
|
177
|
-
do {
|
|
178
|
-
console.log(`Page ${page}: ${results.items.length} jobs.`);
|
|
179
|
-
for (const job of results.items) {
|
|
180
|
-
if (!job[field])
|
|
181
|
-
{
|
|
182
|
-
throw new Error(`Job ${job._id} has no ${field} field`);
|
|
183
|
-
}
|
|
184
|
-
jobsPerField[job[field]] = (jobsPerField[job[field]] || 0) + 1;
|
|
185
|
-
if (field === 'cityText' && !cityLocationAddress[job[field]] && !cityLocations[job[field]]) {
|
|
186
|
-
cityLocationAddress[job[field]] = job.locationAddress;
|
|
187
|
-
cityLocations[job[field]] = job.location;
|
|
188
|
-
}
|
|
189
|
-
}
|
|
190
|
-
if (results.hasNext()) {
|
|
191
|
-
results = await results.next();
|
|
192
|
-
page++;
|
|
193
|
-
}
|
|
194
|
-
} while (results.hasNext());
|
|
195
|
-
let toSave = [];
|
|
158
|
+
|
|
159
|
+
function iterateOverAllJobs(results, field) {
|
|
160
|
+
const jobsPerField = {};
|
|
161
|
+
const cityLocations = {};
|
|
162
|
+
const citylocationAddress = {};
|
|
163
|
+
countJobsPerGivenField(results, field, jobsPerField);
|
|
196
164
|
if (field === 'cityText') {
|
|
197
|
-
|
|
198
|
-
const locAddress = cityLocationAddress[value] || {};
|
|
199
|
-
const loc = cityLocations[value] || {};
|
|
200
|
-
value = normalizeCityName(value);
|
|
201
|
-
|
|
202
|
-
return {
|
|
203
|
-
title: value,
|
|
204
|
-
_id: value.replace(/\s+/g, ''),
|
|
205
|
-
count: amount,
|
|
206
|
-
locationAddress: locAddress,
|
|
207
|
-
country: loc.country,
|
|
208
|
-
city: loc.city,
|
|
209
|
-
};
|
|
210
|
-
});
|
|
211
|
-
}
|
|
212
|
-
else{
|
|
213
|
-
// Prepare array for bulkSave
|
|
214
|
-
toSave = Object.entries(jobsPerField).map(([value, amount]) => ({
|
|
215
|
-
title: value,
|
|
216
|
-
_id: value.replace(/\s+/g, ''),
|
|
217
|
-
count: amount
|
|
218
|
-
}));
|
|
219
|
-
}
|
|
220
|
-
if (toSave.length === 0) {
|
|
221
|
-
console.log('No jobs found.');
|
|
222
|
-
return { success: true, message: 'No jobs to save.' };
|
|
165
|
+
fillCityLocationAndLocationAddress(results, cityLocations,citylocationAddress);
|
|
223
166
|
}
|
|
167
|
+
return { jobsPerField, cityLocations,citylocationAddress };
|
|
168
|
+
}
|
|
224
169
|
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
}
|
|
170
|
+
async function aggregateJobsByFieldToCMS({ field, collection }) {
|
|
171
|
+
console.log(`counting jobs per ${field}.`);
|
|
172
|
+
let results = await getAllPositions();
|
|
173
|
+
const { jobsPerField, cityLocations,citylocationAddress } = iterateOverAllJobs(results, field);
|
|
174
|
+
const toSave = prepareToSaveArray(jobsPerField, cityLocations, field,citylocationAddress);
|
|
175
|
+
if (toSave.length === 0) {
|
|
176
|
+
console.log('No jobs found.');
|
|
177
|
+
return { success: true, message: 'No jobs to save.' };
|
|
178
|
+
}
|
|
179
|
+
try {
|
|
180
|
+
const saveResult = await wixData.bulkSave(collection, toSave);
|
|
181
|
+
console.log(`Saved ${toSave.length} ${field} counts to ${collection}.`);
|
|
182
|
+
return { success: true, saved: toSave.length, result: saveResult };
|
|
183
|
+
} catch (err) {
|
|
184
|
+
console.error(`Error saving jobs per ${field}:`, err);
|
|
185
|
+
return { success: false, error: err.message };
|
|
186
|
+
}
|
|
233
187
|
}
|
|
234
188
|
|
|
235
189
|
async function getJobsWithNoDescriptions() {
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
190
|
+
let jobswithoutdescriptionsQuery = await wixData
|
|
191
|
+
.query('Jobs1')
|
|
192
|
+
.limit(1000)
|
|
193
|
+
.isEmpty('jobDescription')
|
|
194
|
+
.find();
|
|
195
|
+
return jobswithoutdescriptionsQuery;
|
|
239
196
|
}
|
|
240
197
|
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
// Fetch all jobs
|
|
255
|
-
let jobsResults = await wixData.query(COLLECTIONS.JOBS).limit(1000).find();
|
|
256
|
-
let jobsToUpdate = [];
|
|
257
|
-
console.log('jobsResults',jobsResults.items);
|
|
258
|
-
|
|
259
|
-
do {
|
|
260
|
-
for (const job of jobsResults.items) {
|
|
261
|
-
const refId = sourceMap[job[jobField]];
|
|
262
|
-
if (refId) {
|
|
263
|
-
jobsToUpdate.push({
|
|
264
|
-
...job,
|
|
265
|
-
[referenceField]: refId
|
|
266
|
-
});
|
|
267
|
-
}
|
|
268
|
-
}
|
|
269
|
-
if (jobsResults.hasNext()) {
|
|
270
|
-
jobsResults = await jobsResults.next();
|
|
271
|
-
} else {
|
|
272
|
-
break;
|
|
273
|
-
}
|
|
274
|
-
} while (true);
|
|
198
|
+
/**
|
|
199
|
+
* @param {Object} params
|
|
200
|
+
* @param {"city"|"departmentRef"} params.referenceField
|
|
201
|
+
* @param {"cities1"|"AmountOfJobsPerDepartment1"} params.sourceCollection
|
|
202
|
+
* @param {"cityText"|"department"} params.jobField
|
|
203
|
+
*/
|
|
204
|
+
async function referenceJobsToField({ referenceField, sourceCollection, jobField }) {
|
|
205
|
+
// Fetch all source items (cities or departments)
|
|
206
|
+
const sources = await wixData.query(sourceCollection).limit(1000).find();
|
|
207
|
+
const sourceMap = {};
|
|
208
|
+
for (const item of sources.items) {
|
|
209
|
+
sourceMap[item.title] = item._id;
|
|
210
|
+
}
|
|
275
211
|
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
212
|
+
// Fetch all jobs
|
|
213
|
+
let jobsResults = await getAllPositions();
|
|
214
|
+
let jobsToUpdate = [];
|
|
215
|
+
|
|
216
|
+
for (const job of jobsResults) {
|
|
217
|
+
const refId = sourceMap[job[jobField]];
|
|
218
|
+
if (refId) {
|
|
219
|
+
jobsToUpdate.push({
|
|
220
|
+
...job,
|
|
221
|
+
[referenceField]: refId,
|
|
222
|
+
});
|
|
223
|
+
}
|
|
224
|
+
}
|
|
281
225
|
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
processChunk: async (chunk) => {
|
|
288
|
-
await wixData.bulkUpdate(COLLECTIONS.JOBS, chunk);
|
|
289
|
-
}
|
|
290
|
-
});
|
|
226
|
+
// Remove system fields that cannot be updated
|
|
227
|
+
jobsToUpdate = jobsToUpdate.map(job => {
|
|
228
|
+
const { _createdDate, _updatedDate, ...rest } = job;
|
|
229
|
+
return rest;
|
|
230
|
+
});
|
|
291
231
|
|
|
292
|
-
|
|
293
|
-
|
|
232
|
+
// Bulk update in chunks of 1000
|
|
233
|
+
const chunkSize = 1000;
|
|
234
|
+
await chunkedBulkOperation({
|
|
235
|
+
items: jobsToUpdate,
|
|
236
|
+
chunkSize,
|
|
237
|
+
processChunk: async chunk => {
|
|
238
|
+
await wixData.bulkUpdate('Jobs1', chunk);
|
|
239
|
+
},
|
|
240
|
+
});
|
|
294
241
|
|
|
295
|
-
|
|
296
|
-
return jobDetails.actions.applyOnWeb.url;
|
|
242
|
+
return { success: true, updated: jobsToUpdate.length };
|
|
297
243
|
}
|
|
298
244
|
|
|
299
245
|
function fetchJobLocation(jobDetails) {
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
246
|
+
const isZeroLocation =
|
|
247
|
+
jobDetails.location.latitude === '0.0000' && jobDetails.location.longitude === '0.0000';
|
|
248
|
+
const jobLocation = {
|
|
249
|
+
location: isZeroLocation
|
|
250
|
+
? {}
|
|
251
|
+
: {
|
|
252
|
+
latitude: parseFloat(jobDetails.location.latitude),
|
|
253
|
+
longitude: parseFloat(jobDetails.location.longitude),
|
|
304
254
|
},
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
255
|
+
city: jobDetails.location.city,
|
|
256
|
+
country: jobDetails.location.country,
|
|
257
|
+
formatted: [
|
|
258
|
+
jobDetails.location.city,
|
|
259
|
+
jobDetails.location.region,
|
|
260
|
+
jobDetails.location.regionCode,
|
|
261
|
+
jobDetails.location.country,
|
|
262
|
+
]
|
|
263
|
+
.filter(Boolean)
|
|
264
|
+
.join(', '),
|
|
265
|
+
streetAddress: {},
|
|
266
|
+
subdivision: '',
|
|
267
|
+
postalCode: '',
|
|
268
|
+
};
|
|
269
|
+
return jobLocation;
|
|
319
270
|
}
|
|
320
271
|
|
|
321
272
|
|
|
@@ -327,13 +278,14 @@ function getSmartToken() {
|
|
|
327
278
|
})
|
|
328
279
|
.catch((error) => {
|
|
329
280
|
console.error(error);
|
|
281
|
+
throw error;
|
|
330
282
|
});
|
|
331
283
|
}
|
|
332
284
|
|
|
333
285
|
|
|
334
286
|
async function createApiKeyCollectionAndFillIt() {
|
|
335
287
|
console.log("Creating ApiKey collection and filling it with the smart token");
|
|
336
|
-
await createCollectionIfMissing(COLLECTIONS.API_KEY, COLLECTIONS_FIELDS.API_KEY);
|
|
288
|
+
await createCollectionIfMissing(COLLECTIONS.API_KEY, COLLECTIONS_FIELDS.API_KEY,null,'singleItem');
|
|
337
289
|
console.log("Getting the smart token");
|
|
338
290
|
const token = await getSmartToken();
|
|
339
291
|
console.log("token is : ", token);
|
|
@@ -347,9 +299,9 @@ async function createApiKeyCollectionAndFillIt() {
|
|
|
347
299
|
|
|
348
300
|
|
|
349
301
|
module.exports = {
|
|
350
|
-
|
|
351
|
-
|
|
302
|
+
saveJobsDataToCMS,
|
|
303
|
+
saveJobsDescriptionsAndLocationToCMS,
|
|
352
304
|
aggregateJobsByFieldToCMS,
|
|
353
305
|
referenceJobsToField,
|
|
354
306
|
createApiKeyCollectionAndFillIt,
|
|
355
|
-
};
|
|
307
|
+
};
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
const { fetch } = require('wix-fetch');
|
|
2
2
|
const { items: wixData } = require('@wix/data');
|
|
3
3
|
const { COLLECTIONS } = require('./collectionConsts');
|
|
4
|
+
|
|
4
5
|
async function makeSmartRecruitersRequest(path,token) {
|
|
5
6
|
// const baseUrl = 'https://api.smartrecruiters.com'; // PROD
|
|
6
7
|
const baseUrl = 'https://aoxley54.wixstudio.com/external-template/_functions'; // TEST
|
|
@@ -34,7 +35,7 @@ async function fetchPositionsFromSRAPI() {
|
|
|
34
35
|
let allPositions = [];
|
|
35
36
|
let totalFound = 0;
|
|
36
37
|
let nextPageId = null; // Start with no page ID for the first request
|
|
37
|
-
let
|
|
38
|
+
let page = 0;
|
|
38
39
|
const MAX_PAGES = 30 // Safety limit to prevent infinite loops
|
|
39
40
|
const token = await getSmartTokenFromCMS();
|
|
40
41
|
|
|
@@ -42,45 +43,44 @@ async function fetchPositionsFromSRAPI() {
|
|
|
42
43
|
|
|
43
44
|
do {
|
|
44
45
|
try {
|
|
45
|
-
|
|
46
|
-
|
|
46
|
+
page++;
|
|
47
|
+
|
|
47
48
|
// Build the API path - first request has no page parameter, subsequent use nextPageId
|
|
48
49
|
let apiPath = '/jobs?limit=50';
|
|
49
50
|
if (nextPageId) {
|
|
50
51
|
apiPath += `&nextPageId=${nextPageId}`;
|
|
51
52
|
}
|
|
52
53
|
|
|
53
|
-
console.log(`Fetching page ${
|
|
54
|
+
console.log(`Fetching page ${page} with path: ${apiPath}`);
|
|
54
55
|
const response = await makeSmartRecruitersRequest(apiPath,token);
|
|
55
56
|
|
|
56
57
|
// Add positions from this page to our collection
|
|
57
58
|
if (response.content && Array.isArray(response.content)) {
|
|
58
59
|
allPositions = allPositions.concat(response.content);
|
|
59
|
-
console.log(`Page ${
|
|
60
|
+
console.log(`Page ${page}: Found ${response.content.length} positions`);
|
|
60
61
|
}
|
|
61
|
-
|
|
62
|
+
|
|
62
63
|
// Update total count from first response
|
|
63
|
-
if (
|
|
64
|
+
if (page === 1) {
|
|
64
65
|
totalFound = response.totalFound || 0;
|
|
65
66
|
console.log(`Total positions available: ${totalFound}`);
|
|
66
67
|
}
|
|
67
|
-
|
|
68
|
+
|
|
68
69
|
// Get the nextPageId for the next iteration
|
|
69
|
-
nextPageId = response.nextPageId && response.nextPageId !==
|
|
70
|
-
|
|
70
|
+
nextPageId = response.nextPageId && response.nextPageId !== '' ? response.nextPageId : null;
|
|
71
|
+
|
|
71
72
|
if (nextPageId) {
|
|
72
73
|
console.log(`Next page ID: ${nextPageId}`);
|
|
73
74
|
} else {
|
|
74
75
|
console.log('No more pages to fetch');
|
|
75
76
|
}
|
|
76
|
-
|
|
77
77
|
} catch (error) {
|
|
78
|
-
console.error(`Error fetching page ${
|
|
78
|
+
console.error(`Error fetching page ${page}:`, error);
|
|
79
79
|
throw error;
|
|
80
80
|
}
|
|
81
|
-
|
|
81
|
+
|
|
82
82
|
// Safety check to prevent infinite loops
|
|
83
|
-
if (
|
|
83
|
+
if (page >= MAX_PAGES) {
|
|
84
84
|
console.warn(`Reached maximum page limit of ${MAX_PAGES}. Stopping pagination.`);
|
|
85
85
|
break;
|
|
86
86
|
}
|
|
@@ -89,12 +89,12 @@ async function fetchPositionsFromSRAPI() {
|
|
|
89
89
|
console.log(`Finished fetching all pages. Total positions collected: ${allPositions.length}`);
|
|
90
90
|
|
|
91
91
|
// Return response in the same format as before, but with all positions
|
|
92
|
-
const result =
|
|
92
|
+
const result = {
|
|
93
93
|
totalFound: totalFound,
|
|
94
94
|
offset: 0,
|
|
95
95
|
limit: allPositions.length,
|
|
96
|
-
nextPageId:
|
|
97
|
-
content: allPositions
|
|
96
|
+
nextPageId: '', // Always empty since we've fetched everything
|
|
97
|
+
content: allPositions,
|
|
98
98
|
};
|
|
99
99
|
|
|
100
100
|
const amountOfUniqueJobs = new Set(allPositions.map(job => job.id)).size;
|
|
@@ -108,8 +108,7 @@ async function fetchPositionsFromSRAPI() {
|
|
|
108
108
|
}
|
|
109
109
|
|
|
110
110
|
async function fetchJobDescription(jobId) {
|
|
111
|
-
|
|
112
|
-
return await makeSmartRecruitersRequest(`/jobs/${jobId}`,token);
|
|
111
|
+
return await makeSmartRecruitersRequest(`/jobs/${jobId}`);
|
|
113
112
|
}
|
|
114
113
|
|
|
115
114
|
async function getSmartTokenFromCMS() {
|
|
@@ -117,12 +116,12 @@ async function getSmartTokenFromCMS() {
|
|
|
117
116
|
if (result.items.length > 0) {
|
|
118
117
|
return result.items[0].token; // This is your string token
|
|
119
118
|
} else {
|
|
120
|
-
|
|
119
|
+
throw new Error('[getSmartTokenFromCMS], No token found');
|
|
121
120
|
}
|
|
122
121
|
}
|
|
123
122
|
|
|
124
123
|
|
|
125
124
|
module.exports = {
|
|
126
|
-
|
|
127
|
-
|
|
125
|
+
fetchPositionsFromSRAPI,
|
|
126
|
+
fetchJobDescription,
|
|
128
127
|
};
|
package/backend/index.js
CHANGED
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
module.exports = {
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
2
|
+
...require('./utils'),
|
|
3
|
+
...require('./queries'),
|
|
4
|
+
...require('./fetchPositionsFromSRAPI'),
|
|
5
|
+
...require('./consts'),
|
|
6
|
+
};
|
package/backend/queries.js
CHANGED
|
@@ -1,19 +1,22 @@
|
|
|
1
|
-
const {
|
|
2
|
-
const { items } = require('@wix/data');
|
|
3
|
-
|
|
1
|
+
const { items: wixData } = require('@wix/data');
|
|
4
2
|
|
|
5
3
|
async function getAllPositions() {
|
|
6
|
-
|
|
7
|
-
|
|
4
|
+
return wixData
|
|
5
|
+
.query('Jobs1')
|
|
6
|
+
.limit(1000)
|
|
7
|
+
.find()
|
|
8
|
+
.then(result => result.items);
|
|
8
9
|
}
|
|
9
10
|
|
|
10
11
|
async function getPositionsByField(field, value) {
|
|
11
|
-
|
|
12
|
-
|
|
12
|
+
return wixData
|
|
13
|
+
.query('Jobs1')
|
|
14
|
+
.eq(field, value)
|
|
15
|
+
.find()
|
|
16
|
+
.then(result => result.items);
|
|
13
17
|
}
|
|
14
18
|
|
|
15
|
-
|
|
16
19
|
module.exports = {
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
};
|
|
20
|
+
getAllPositions,
|
|
21
|
+
getPositionsByField,
|
|
22
|
+
};
|
package/backend/utils.js
CHANGED
|
@@ -1,11 +1,70 @@
|
|
|
1
1
|
async function chunkedBulkOperation({ items, chunkSize, processChunk }) {
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
2
|
+
for (let i = 0; i < items.length; i += chunkSize) {
|
|
3
|
+
const chunk = items.slice(i, i + chunkSize);
|
|
4
|
+
await processChunk(chunk, Math.floor(i / chunkSize) + 1);
|
|
5
|
+
}
|
|
6
|
+
}
|
|
7
|
+
|
|
8
|
+
async function delay(ms) {
|
|
9
|
+
await new Promise(resolve => setTimeout(resolve, ms));
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
function countJobsPerGivenField(jobs, field,jobsPerField) {
|
|
13
|
+
for (const job of jobs) {
|
|
14
|
+
if (!job[field]) {
|
|
15
|
+
throw new Error(`Job ${job._id} has no ${field} field`);
|
|
16
|
+
}
|
|
17
|
+
jobsPerField[job[field]] = (jobsPerField[job[field]] || 0) + 1;
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
function fillCityLocationAndLocationAddress(jobs, cityLocations,citylocationAddress) {
|
|
22
|
+
for (const job of jobs) {
|
|
23
|
+
if (!cityLocations[job.cityText] && !citylocationAddress[job.cityText]) {
|
|
24
|
+
cityLocations[job.cityText] = job.location;
|
|
25
|
+
citylocationAddress[job.cityText] = job.locationAddress;
|
|
5
26
|
}
|
|
27
|
+
}
|
|
6
28
|
}
|
|
7
29
|
|
|
30
|
+
function prepareToSaveArray(jobsPerField, cityLocations, field,citylocationAddress) {
|
|
31
|
+
if (field === 'cityText') {
|
|
32
|
+
return Object.entries(jobsPerField).map(([value, amount]) => {
|
|
33
|
+
const loc = cityLocations[value] || {};
|
|
34
|
+
const locAddress = citylocationAddress[value] || {};
|
|
35
|
+
value = normalizeCityName(value);
|
|
36
|
+
return {
|
|
37
|
+
title: value,
|
|
38
|
+
_id: value.replace(/\s+/g, ''),
|
|
39
|
+
count: amount,
|
|
40
|
+
locationAddress: locAddress,
|
|
41
|
+
country: loc.country,
|
|
42
|
+
city: loc.city,
|
|
43
|
+
};
|
|
44
|
+
});
|
|
45
|
+
} else {
|
|
46
|
+
return Object.entries(jobsPerField).map(([value, amount]) => ({
|
|
47
|
+
title: value,
|
|
48
|
+
_id: value.replace(/\s+/g, ''),
|
|
49
|
+
count: amount,
|
|
50
|
+
}));
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
function normalizeCityName(city) {
|
|
55
|
+
if (!city) return city;
|
|
56
|
+
// Remove accents/diacritics, trim whitespace
|
|
57
|
+
return city
|
|
58
|
+
.normalize('NFD')
|
|
59
|
+
.replace(/\p{Diacritic}/gu, '')
|
|
60
|
+
.trim();
|
|
61
|
+
}
|
|
8
62
|
|
|
9
63
|
module.exports = {
|
|
10
|
-
|
|
11
|
-
|
|
64
|
+
chunkedBulkOperation,
|
|
65
|
+
delay,
|
|
66
|
+
countJobsPerGivenField,
|
|
67
|
+
fillCityLocationAndLocationAddress,
|
|
68
|
+
prepareToSaveArray,
|
|
69
|
+
normalizeCityName,
|
|
70
|
+
};
|
package/index.js
CHANGED
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "sr-npm",
|
|
3
|
-
"version": "1.7.
|
|
3
|
+
"version": "1.7.66",
|
|
4
4
|
"description": "",
|
|
5
5
|
"main": "index.js",
|
|
6
6
|
"scripts": {
|
|
@@ -17,9 +17,12 @@
|
|
|
17
17
|
},
|
|
18
18
|
"homepage": "https://github.com/psdevteamenterprise/sr-npm#readme",
|
|
19
19
|
"dependencies": {
|
|
20
|
-
"@hisense-staging/velo-npm": "
|
|
20
|
+
"@hisense-staging/velo-npm": "3.0.19",
|
|
21
21
|
"@wix/data": "^1.0.211",
|
|
22
22
|
"@wix/essentials": "^0.1.24",
|
|
23
|
-
"@wix/secrets": "1.0.53"
|
|
23
|
+
"@wix/secrets": "^1.0.53"
|
|
24
|
+
},
|
|
25
|
+
"devDependencies": {
|
|
26
|
+
"prettier": "^3.6.2"
|
|
24
27
|
}
|
|
25
28
|
}
|
package/pages/index.js
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
const {
|
|
2
|
+
htmlToText,
|
|
3
|
+
} = require('../public/utils');
|
|
4
|
+
|
|
5
|
+
async function positionPageOnReady(_$w) {
|
|
6
|
+
|
|
7
|
+
bind(_$w);
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
async function bind(_$w) {
|
|
11
|
+
_$w('#datasetJobsItem').onReady(async () => {
|
|
12
|
+
|
|
13
|
+
const item = await _$w('#datasetJobsItem').getCurrentItem();
|
|
14
|
+
|
|
15
|
+
_$w('#companyDescriptionText').text = htmlToText(item.jobDescription.companyDescription.text);
|
|
16
|
+
_$w('#responsibilitiesText').text = htmlToText(item.jobDescription.jobDescription.text);
|
|
17
|
+
_$w('#qualificationsText').text = htmlToText(item.jobDescription.qualifications.text);
|
|
18
|
+
_$w('#relatedJobsTitleText').text = `More ${item.department} Positions`;
|
|
19
|
+
});
|
|
20
|
+
|
|
21
|
+
_$w('#relatedJobsDataset').onReady(async () => {
|
|
22
|
+
const count = await _$w('#relatedJobsDataset').getTotalCount();
|
|
23
|
+
if(!count){
|
|
24
|
+
_$w('#relatedJobsSection').collapse();
|
|
25
|
+
}
|
|
26
|
+
});
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
module.exports = {
|
|
32
|
+
positionPageOnReady,
|
|
33
|
+
};
|
|
34
|
+
|
package/public/index.js
ADDED
package/public/utils.js
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
|
|
2
|
+
|
|
3
|
+
function htmlToText(html) {
|
|
4
|
+
if (!html) return '';
|
|
5
|
+
|
|
6
|
+
// Remove HTML tags and decode entities
|
|
7
|
+
let text = html
|
|
8
|
+
.replace(/<br\s*\/?>/gi, '\n')
|
|
9
|
+
.replace(/<\/?(p|div|h[1-6])\s*\/?>/gi, '\n')
|
|
10
|
+
.replace(/<li[^>]*>/gi, '• ')
|
|
11
|
+
.replace(/<\/li>/gi, '\n')
|
|
12
|
+
.replace(/<[^>]*>/g, '')
|
|
13
|
+
.replace(/&/g, '&')
|
|
14
|
+
.replace(/</g, '<')
|
|
15
|
+
.replace(/>/g, '>')
|
|
16
|
+
.replace(/"/g, '"')
|
|
17
|
+
.replace(/'/g, "'")
|
|
18
|
+
.replace(/ /g, ' ');
|
|
19
|
+
|
|
20
|
+
console.log(text);
|
|
21
|
+
// Clean up whitespace
|
|
22
|
+
return text.replace(/\n\s*\n+/g, '\n\n').replace(/[ \t]+\n/g, '\n').trim();
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
module.exports = {
|
|
27
|
+
htmlToText
|
|
28
|
+
};
|