tango-app-api-analysis-traffic 3.8.7-vms.31 → 3.8.7-vms.32

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "tango-app-api-analysis-traffic",
3
- "version": "3.8.7-vms.31",
3
+ "version": "3.8.7-vms.32",
4
4
  "description": "Traffic Analysis",
5
5
  "main": "index.js",
6
6
  "type": "module",
@@ -1,7 +1,7 @@
1
1
  import { logger, insertOpenSearchData, getOpenSearchData, updateOpenSearchData } from 'tango-app-api-middleware';
2
2
  import { findOnerevopConfig } from '../services/revopConfig.service.js';
3
3
  import * as clientService from '../services/clients.services.js';
4
- import { bulkUpdate, insertWithId, searchOpenSearchData, scrollResponse, upsertWithScript } from 'tango-app-api-middleware/src/utils/openSearch.js';
4
+ import { bulkUpdate, insertWithId, scrollResponse, searchOpenSearchData, upsertWithScript } from 'tango-app-api-middleware/src/utils/openSearch.js';
5
5
  import { findOneVmsStoreRequest } from '../services/vmsStoreRequest.service.js';
6
6
  // import dayjs from 'dayjs';
7
7
  // Lamda Service Call //
@@ -70,7 +70,7 @@ export async function revoptagging( req, res ) {
70
70
 
71
71
  let respo= await getOpenSearchData( openSearch.revops, searchQuery );
72
72
  const revopData = respo?.body?.hits?.hits;
73
- if ( revopData&& revopData.length>0 ) {
73
+ if ( revopData && revopData.length>0 ) {
74
74
  await updateOpenSearchData( openSearch.revops, revopData[0]._id, { doc: item } );
75
75
  } else {
76
76
  item.createdAt = new Date();
@@ -140,14 +140,29 @@ export async function getrevoptagging( req, res ) {
140
140
 
141
141
  export async function migrateRevopIndex( req, res ) {
142
142
  try {
143
- const { storeId, dateString, size = 500 } = req.body;
143
+ const { storeId, dateString, size = 100 } = req.body;
144
144
  const openSearch = JSON.parse( process.env.OPENSEARCH );
145
145
 
146
146
  const query = {
147
147
  size: size,
148
148
  query: {
149
149
  bool: {
150
- must: [],
150
+ must: [
151
+ // {
152
+ // range: {
153
+ // createdAt: {
154
+ // gte: '2025-10-01T00:00:00.000Z',
155
+ // lte: '2025-10-31T23:59:59.000Z',
156
+ // },
157
+ // },
158
+ // },
159
+ {
160
+ term: {
161
+ 'type.keyword': 'tagging-reflect',
162
+
163
+ },
164
+ },
165
+ ],
151
166
  },
152
167
  },
153
168
  };
@@ -168,21 +183,80 @@ export async function migrateRevopIndex( req, res ) {
168
183
  } );
169
184
  }
170
185
 
171
- const response = await getOpenSearchData( openSearch.revop, query );
172
- const hits = response?.body?.hits?.hits || [];
186
+ // const response = await getOpenSearchData( openSearch.revop, query );
187
+ // const hits = response?.body?.hits?.hits || [];
173
188
 
174
- if ( hits.length === 0 ) {
175
- return res.sendSuccess( { message: 'No records found for migration', updated: 0 } );
189
+ // Use OpenSearch scroll API to retrieve up to 60000 records efficiently
190
+ let allHits = [];
191
+ let scrollId = null;
192
+ let totalFetched = 0;
193
+ let firstResponse = await searchOpenSearchData( openSearch.revop, query );
194
+ // Collect first batch
195
+ let hitsBatch = firstResponse?.body?.hits?.hits || [];
196
+ if ( hitsBatch.length > 0 ) {
197
+ allHits.push( ...hitsBatch );
198
+ totalFetched += hitsBatch.length;
199
+ scrollId = firstResponse.body._scroll_id;
176
200
  }
177
201
 
178
202
  const bulkBody = [];
179
-
180
- for ( const hit of hits ) {
203
+ for ( const hit of hitsBatch ) {
181
204
  const src = hit._source || {};
182
205
  const statusValue = ( src.status || '' ).toLowerCase();
183
- const parentValue = src.parent;
184
- const idValue = `${src.storeId || ''}_${src.dateString || src.dteString || ''}_${src.tempId || ''}`;
206
+ // const parentValue = src.parent;
207
+
208
+ // Get ticket sattasu from the footfalldirectory index matching src.storeId and src.dateString
209
+ let ticketStatus = null;
210
+
211
+ const footfallQuery = {
212
+ size: 1,
213
+ query: {
214
+ bool: {
215
+ must: [
216
+ { term: { 'storeId.keyword': src.storeId } },
217
+ { term: { 'dateString': src.dateString } },
218
+ ],
219
+ },
220
+ },
221
+ };
222
+ const footfallResp = await getOpenSearchData( openSearch.oldFootfallDirectory, footfallQuery );
223
+ ticketStatus = footfallResp?.body?.hits?.hits?.[0]?._source?.status || null;
224
+ if ( src?.duplicateImage?.length > 0 ) {
225
+ src.duplicateImage = src.duplicateImage.map( ( item ) => ( {
226
+ ...item,
227
+ id: `${src.storeId || ''}_${src.dateString || src.dteString || ''}_${item.tempId || ''}`,
228
+ actions: ( ticketStatus === 'closed' && item.isChecked === true ) ? [
229
+ {
230
+ actionType: 'tagging',
231
+ action: 'submitted',
232
+ },
233
+ {
234
+ actionType: 'review',
235
+ action: 'approved',
236
+ },
237
+ ]: ( ticketStatus === 'closed' && item.isChecked === false )?
238
+ [
239
+ {
240
+ actionType: 'tagging',
241
+ action: 'submitted',
242
+ },
243
+ {
244
+ actionType: 'review',
245
+ action: 'rejected',
246
+ },
247
+ ]:
248
+ [
249
+ {
250
+ actionType: 'tagging',
251
+ action: 'submitted',
252
+ },
253
+ ],
254
+ // Include relevant action, assuming 'actions' will be determined below
255
+ } ) );
256
+ }
257
+
185
258
 
259
+ const idValue = `${src.storeId || ''}_${src.dateString || src.dteString || ''}_${src.tempId || ''}`;
186
260
  let actions = [
187
261
  {
188
262
  actionType: 'tagging',
@@ -214,12 +288,15 @@ export async function migrateRevopIndex( req, res ) {
214
288
  ];
215
289
  }
216
290
 
291
+
217
292
  const doc = {
293
+ ...src,
218
294
  id: idValue,
219
- isParent: parentValue === null || parentValue === undefined ? false : true,
295
+ revopsType: src?.revopsType === 'house-keeping'? 'houseKeeping' : src?.revopsType,
296
+ isParent: src?.duplicateImage?.length > 0? true : false,
220
297
  actions,
221
298
  ticketStatus: src.status,
222
- // updatedAt: new Date(),
299
+ // updatedAt: new Date(),
223
300
  };
224
301
 
225
302
  bulkBody.push(
@@ -229,11 +306,274 @@ export async function migrateRevopIndex( req, res ) {
229
306
  }
230
307
 
231
308
  const bulkRes = await bulkUpdate( bulkBody );
309
+
232
310
  if ( bulkRes?.errors ) {
233
311
  logger.error( 'Bulk migration errors:', bulkRes.items );
234
312
  return res.sendError( 'Failed to migrate some records', 500 );
235
313
  }
236
314
 
315
+ while ( hitsBatch.length > 0 && scrollId ) {
316
+ // Fetch next batch using scroll_id
317
+ const nextScrollRes = await scrollResponse( scrollId );
318
+
319
+ hitsBatch = nextScrollRes?.body?.hits?.hits || [];
320
+ if ( hitsBatch.length === 0 ) break;
321
+ logger.info( { hitsBatch: hitsBatch?.length } );
322
+ const bulkBody = [];
323
+ for ( const hit of hitsBatch ) {
324
+ const src = hit._source || {};
325
+ const statusValue = ( src.status || '' ).toLowerCase();
326
+ // const parentValue = src.parent;
327
+
328
+ // Get ticket sattasu from the footfalldirectory index matching src.storeId and src.dateString
329
+ let ticketStatus = null;
330
+
331
+ const footfallQuery = {
332
+ size: 1,
333
+ query: {
334
+ bool: {
335
+ must: [
336
+ { term: { 'storeId.keyword': src.storeId } },
337
+ { term: { 'dateString': src.dateString } },
338
+ ],
339
+ },
340
+ },
341
+ };
342
+ const footfallResp = await getOpenSearchData( openSearch.oldFootfallDirectory, footfallQuery );
343
+ ticketStatus = footfallResp?.body?.hits?.hits?.[0]?._source?.status || null;
344
+ if ( src?.duplicateImage?.length > 0 ) {
345
+ src.duplicateImage = src.duplicateImage.map( ( item ) => ( {
346
+ ...item,
347
+ id: `${src.storeId || ''}_${src.dateString || src.dteString || ''}_${item.tempId || ''}`,
348
+ actions: ( ticketStatus === 'closed' && item.isChecked === true ) ? [
349
+ {
350
+ actionType: 'tagging',
351
+ action: 'submitted',
352
+ },
353
+ {
354
+ actionType: 'review',
355
+ action: 'approved',
356
+ },
357
+ ]: ( ticketStatus === 'closed' && item.isChecked === false )?
358
+ [
359
+ {
360
+ actionType: 'tagging',
361
+ action: 'submitted',
362
+ },
363
+ {
364
+ actionType: 'review',
365
+ action: 'rejected',
366
+ },
367
+ ]:
368
+ [
369
+ {
370
+ actionType: 'tagging',
371
+ action: 'submitted',
372
+ },
373
+ ],
374
+ // Include relevant action, assuming 'actions' will be determined below
375
+ } ) );
376
+ }
377
+
378
+
379
+ const idValue = `${src.storeId || ''}_${src.dateString || src.dteString || ''}_${src.tempId || ''}`;
380
+ logger.info( { idValue } );
381
+ let actions = [
382
+ {
383
+ actionType: 'tagging',
384
+ action: 'submitted',
385
+ },
386
+ ];
387
+
388
+ if ( statusValue === 'approved' ) {
389
+ actions = [
390
+ {
391
+ actionType: 'tagging',
392
+ action: 'submitted',
393
+ },
394
+ {
395
+ actionType: 'review',
396
+ action: 'approved',
397
+ },
398
+ ];
399
+ } else if ( statusValue === 'rejected' ) {
400
+ actions = [
401
+ {
402
+ actionType: 'tagging',
403
+ action: 'submitted',
404
+ },
405
+ {
406
+ actionType: 'review',
407
+ action: 'rejected',
408
+ },
409
+ ];
410
+ }
411
+
412
+
413
+ const doc = {
414
+ ...src,
415
+ id: idValue,
416
+ revopsType: src?.revopsType === 'house-keeping'? 'houseKeeping' : src?.revopsType,
417
+ isParent: src?.duplicateImage?.length > 0? true : false,
418
+ actions,
419
+ ticketStatus: src.status,
420
+ // updatedAt: new Date(),
421
+ };
422
+
423
+ bulkBody.push(
424
+ { update: { _index: openSearch.newRevop, _id: hit._id } },
425
+ { doc: doc, doc_as_upsert: true },
426
+ );
427
+ }
428
+
429
+ const bulkRes = await bulkUpdate( bulkBody );
430
+
431
+ if ( bulkRes?.errors ) {
432
+ logger.error( 'Bulk migration errors:', bulkRes.items );
433
+ return res.sendError( 'Failed to migrate some records', 500 );
434
+ }
435
+ allHits.push( ...hitsBatch );
436
+ totalFetched += hitsBatch.length;
437
+ logger.info( { totalFetched } );
438
+ // Protect against exceeding limit
439
+
440
+ scrollId = nextScrollRes.body._scroll_id;
441
+ }
442
+
443
+ // For downstream logic, use allHits instead of hits
444
+ const hits = allHits;
445
+
446
+ if ( hits.length === 0 ) {
447
+ return res.sendSuccess( { message: 'No records found for migration', updated: 0 } );
448
+ }
449
+
450
+
451
+ // for ( const hit of hits ) {
452
+ // const src = hit._source || {};
453
+ // const statusValue = ( src.status || '' ).toLowerCase();
454
+ // // const parentValue = src.parent;
455
+
456
+ // // Get ticket sattasu from the footfalldirectory index matching src.storeId and src.dateString
457
+ // let ticketStatus = null;
458
+
459
+ // const footfallQuery = {
460
+ // size: 1,
461
+ // query: {
462
+ // bool: {
463
+ // must: [
464
+ // { term: { 'storeId.keyword': src.storeId } },
465
+ // { term: { 'dateString': src.dateString } },
466
+ // ],
467
+ // },
468
+ // },
469
+ // };
470
+ // const footfallResp = await getOpenSearchData( openSearch.oldFootfallDirectory, footfallQuery );
471
+ // ticketStatus = footfallResp?.body?.hits?.hits?.[0]?._source?.status || null;
472
+ // if ( src?.duplicateImage?.length > 0 ) {
473
+ // src.duplicateImage = src.duplicateImage.map( ( item ) => ( {
474
+ // ...item,
475
+ // id: `${src.storeId || ''}_${src.dateString || src.dteString || ''}_${item.tempId || ''}`,
476
+ // actions: ( ticketStatus === 'closed' && item.isChecked === true ) ? [
477
+ // {
478
+ // actionType: 'tagging',
479
+ // action: 'submitted',
480
+ // },
481
+ // {
482
+ // actionType: 'review',
483
+ // action: 'approved',
484
+ // },
485
+ // ]: ( ticketStatus === 'closed' && item.isChecked === false )?
486
+ // [
487
+ // {
488
+ // actionType: 'tagging',
489
+ // action: 'submitted',
490
+ // },
491
+ // {
492
+ // actionType: 'review',
493
+ // action: 'rejected',
494
+ // },
495
+ // ]:
496
+ // [
497
+ // {
498
+ // actionType: 'tagging',
499
+ // action: 'submitted',
500
+ // },
501
+ // ],
502
+ // // Include relevant action, assuming 'actions' will be determined below
503
+ // } ) );
504
+ // }
505
+
506
+
507
+ // const idValue = `${src.storeId || ''}_${src.dateString || src.dteString || ''}_${src.tempId || ''}`;
508
+
509
+ // let actions = [
510
+ // {
511
+ // actionType: 'tagging',
512
+ // action: 'submitted',
513
+ // },
514
+ // ];
515
+
516
+ // if ( statusValue === 'approved' ) {
517
+ // actions = [
518
+ // {
519
+ // actionType: 'tagging',
520
+ // action: 'submitted',
521
+ // },
522
+ // {
523
+ // actionType: 'review',
524
+ // action: 'approved',
525
+ // },
526
+ // ];
527
+ // } else if ( statusValue === 'rejected' ) {
528
+ // actions = [
529
+ // {
530
+ // actionType: 'tagging',
531
+ // action: 'submitted',
532
+ // },
533
+ // {
534
+ // actionType: 'review',
535
+ // action: 'rejected',
536
+ // },
537
+ // ];
538
+ // }
539
+
540
+
541
+ // const doc = {
542
+ // ...src,
543
+ // id: idValue,
544
+ // revopsType: src?.revopsType === 'house-keeping'? 'houseKeeping' : src?.revopsType,
545
+ // isParent: src?.duplicateImage?.length > 0? true : false,
546
+ // actions,
547
+ // ticketStatus: src.status,
548
+ // // updatedAt: new Date(),
549
+ // };
550
+
551
+
552
+ // bulkBody.push(
553
+ // { update: { _index: openSearch.newRevop, _id: hit._id } },
554
+ // { doc: doc, doc_as_upsert: true },
555
+ // );
556
+ // }
557
+ // Implement batch by batch update
558
+ // const BATCH_SIZE = 10000; // You can adjust the batch size as needed
559
+
560
+ // for ( let i = 0; i < bulkBody.length; i += BATCH_SIZE ) {
561
+ // const batch = bulkBody.slice( i, i + BATCH_SIZE );
562
+ // const bulkRes = await bulkUpdate( batch );
563
+
564
+ // if ( bulkRes?.errors ) {
565
+ // logger.error( 'Bulk migration errors:', bulkRes.items );
566
+ // return res.sendError( 'Failed to migrate some records', 500 );
567
+ // }
568
+ // }
569
+
570
+ // const bulkRes = await bulkUpdate( bulkBody );
571
+
572
+ // if ( bulkRes?.errors ) {
573
+ // logger.error( 'Bulk migration errors:', bulkRes.items );
574
+ // return res.sendError( 'Failed to migrate some records', 500 );
575
+ // }
576
+
237
577
  return res.sendSuccess( { message: 'Migration completed', updated: hits.length } );
238
578
  } catch ( error ) {
239
579
  logger.error( { error: error, message: req.body, function: 'migrateRevopIndex' } );
@@ -1002,7 +1342,7 @@ export async function vmsDataMigration( req, res ) {
1002
1342
  try {
1003
1343
  const openSearch = JSON.parse( process.env.OPENSEARCH );
1004
1344
  const inputData = req.body;
1005
- const { storeId, dateString, limit = 100 } = inputData;
1345
+ const { storeId, dateString, limit = 10000 } = inputData;
1006
1346
 
1007
1347
  // Build query to fetch old structure documents
1008
1348
  const query = {
@@ -1014,6 +1354,14 @@ export async function vmsDataMigration( req, res ) {
1014
1354
  'ticketName.keyword': 'footfall-directory',
1015
1355
  },
1016
1356
  },
1357
+ // {
1358
+ // range: {
1359
+ // createdAt: {
1360
+ // gte: '2025-12-01T00:00:00.000Z',
1361
+ // lte: '2025-12-03T00:00:00.000Z',
1362
+ // },
1363
+ // },
1364
+ // },
1017
1365
  ],
1018
1366
  },
1019
1367
  },
@@ -1069,17 +1417,22 @@ export async function vmsDataMigration( req, res ) {
1069
1417
  const documentId = hit._id;
1070
1418
 
1071
1419
  // Calculate revicedFootfall (sum of AC counts)
1072
- const revicedFootfall = ( oldSource.duplicateACCount || 0 ) +
1420
+ const tempFootfall =oldSource?.status === 'open' ?
1421
+ ( oldSource.duplicateCount || 0 ) +
1422
+ ( oldSource.employeeCount || 0 ) +
1423
+ ( oldSource.houseKeepingCount || 0 ) +
1424
+ ( oldSource.junkCount || 0 ):
1425
+ ( oldSource.duplicateACCount || 0 ) +
1073
1426
  ( oldSource.employeeACCount || 0 ) +
1074
1427
  ( oldSource.houseKeepingACCount || 0 ) +
1075
1428
  ( oldSource.junkACCount || 0 );
1076
1429
 
1077
1430
  // Calculate revicedPerc
1078
1431
  const footfallCount = oldSource.footfallCount || 0;
1432
+ const revicedFootfall = footfallCount - tempFootfall;
1079
1433
  const revicedPerc = footfallCount > 0 ?
1080
1434
  Math.round( ( revicedFootfall / footfallCount ) * 100 ) :
1081
1435
  0;
1082
-
1083
1436
  // Calculate reviced
1084
1437
  const reviced = parseInt( revicedPerc );
1085
1438
 
@@ -1269,7 +1622,8 @@ export async function vmsDataMigration( req, res ) {
1269
1622
  ];
1270
1623
 
1271
1624
  // Create mappingInfo array
1272
- const mappingInfo = [
1625
+ const mappingInfo = oldSource.status === 'open' ?
1626
+ [
1273
1627
  {
1274
1628
  type: 'tagging',
1275
1629
  mode: 'mobile',
@@ -1290,9 +1644,41 @@ export async function vmsDataMigration( req, res ) {
1290
1644
  revisedDetail,
1291
1645
  status: oldSource.status === 'open' ? 'Open' : oldSource.status || 'Open',
1292
1646
  dueDate: oldSource.updatedAt ? new Date( new Date( oldSource.updatedAt ).getTime() + 3 * 24 * 60 * 60 * 1000 ) : new Date( Date.now() + 3 * 24 * 60 * 60 * 1000 ),
1647
+ createdAt: oldSource.createdAt || new Date(),
1293
1648
  },
1294
- ];
1295
-
1649
+ ]:
1650
+
1651
+ [
1652
+ {
1653
+ type: 'tagging',
1654
+ mode: 'web',
1655
+ revicedFootfall,
1656
+ revicedPerc: `${revicedPerc}%`,
1657
+ reviced,
1658
+ count,
1659
+ revisedDetail,
1660
+ status: oldSource.status === 'closed' ? 'Closed' : oldSource.status || 'Closed',
1661
+ createdByEmail: oldSource.email || '',
1662
+ createdByUserName: oldSource.userName || '',
1663
+ createdByRole: oldSource.role || 'user',
1664
+ createdAt: oldSource.createdAt || new Date(),
1665
+ },
1666
+ {
1667
+ type: 'review',
1668
+ mode: 'web',
1669
+ revicedFootfall,
1670
+ revicedPerc: `${revicedPerc}%`,
1671
+ reviced,
1672
+ count,
1673
+ revisedDetail,
1674
+ status: oldSource.status === 'closed' ? 'Closed' : oldSource.status || 'Closed',
1675
+ dueDate: oldSource.createdAt ? new Date( new Date( oldSource.createdAt ).getTime() + 3 * 24 * 60 * 60 * 1000 ) : new Date( Date.now() + 3 * 24 * 60 * 60 * 1000 ),
1676
+ createdAt: oldSource.createdAt || new Date(),
1677
+ createdByEmail: oldSource.approverEmail || '',
1678
+ createdByUserName: oldSource.approverUserName || '',
1679
+ createdByRole: oldSource.approverRole || 'user',
1680
+ },
1681
+ ];
1296
1682
  // Create new structure
1297
1683
  const newSource = {
1298
1684
  storeId: oldSource.storeId,
@@ -1305,7 +1691,7 @@ export async function vmsDataMigration( req, res ) {
1305
1691
  ticketId: oldSource.ticketId,
1306
1692
  createdAt: oldSource.createdAt,
1307
1693
  updatedAt: oldSource.updatedAt,
1308
- status: oldSource.status === 'open' ? 'Raised' : oldSource.status || 'Raised',
1694
+ status: oldSource.status === 'open' ? 'Raised' : 'Closed' || 'Raised',
1309
1695
  comments: oldSource.comments || '',
1310
1696
  revicedFootfall,
1311
1697
  revicedPerc: `${revicedPerc}%`,
@@ -1314,8 +1700,9 @@ export async function vmsDataMigration( req, res ) {
1314
1700
  };
1315
1701
 
1316
1702
  // Update document in OpenSearch
1317
- // await updateOpenSearchData( openSearch.footfallDirectory, documentId, { doc: newSource } );
1318
- // migratedCount++;
1703
+ const updatedData = await updateOpenSearchData( openSearch.footfallDirectory, documentId, { doc: newSource, doc_as_upsert: true } );
1704
+ logger.info( { updatedData } );
1705
+ migratedCount++;
1319
1706
 
1320
1707
  logger.info( { message: 'Document migrated successfully', newSource, documentId, storeId: oldSource.storeId, dateString: oldSource.dateString } );
1321
1708
  } catch ( error ) {