xcraft-core-utils 4.3.5 → 4.3.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.editorconfig +9 -9
- package/.eslintrc.js +28 -28
- package/.zou-flow +3 -0
- package/README.md +3 -3
- package/index.js +24 -24
- package/lib/.babelrc +8 -8
- package/lib/arrayCollector.js +33 -33
- package/lib/async.js +34 -34
- package/lib/batch.js +24 -24
- package/lib/crypto.js +92 -92
- package/lib/cursorPump.js +29 -29
- package/lib/eventDebouncer.js +21 -21
- package/lib/file-crypto.js +41 -41
- package/lib/files.js +144 -144
- package/lib/job-queue.js +113 -114
- package/lib/js.js +14 -14
- package/lib/json.js +12 -12
- package/lib/locks.js +106 -106
- package/lib/log.js +182 -182
- package/lib/modules.js +184 -188
- package/lib/os.js +13 -13
- package/lib/prop-types.js +154 -154
- package/lib/ranked-cache.js +87 -87
- package/lib/reflect.js +12 -12
- package/lib/regex.js +24 -24
- package/lib/runnerInstance.js +135 -134
- package/lib/string.js +15 -15
- package/lib/whereIs.js +13 -13
- package/lib/yaml.js +10 -10
- package/package.json +53 -53
- package/test/index.js +68 -68
- package/test/jobqueue.js +33 -33
package/lib/file-crypto.js
CHANGED
|
@@ -1,41 +1,41 @@
|
|
|
1
|
-
'use strict';
|
|
2
|
-
|
|
3
|
-
const watt = require('gigawatts');
|
|
4
|
-
const crypto = require('crypto');
|
|
5
|
-
const fs = require('fs');
|
|
6
|
-
|
|
7
|
-
/**
|
|
8
|
-
* Compute the checksum of the file given as parameter using a specified algorithm.
|
|
9
|
-
*
|
|
10
|
-
* @param {string} filePath - The file path.
|
|
11
|
-
* @param {Object} options - Options.
|
|
12
|
-
* @param {string} options.algorithm - For possible values, see https://nodejs.org/api/crypto.html#crypto_crypto_createhash_algorithm_options
|
|
13
|
-
* @param {string} options.encoding - (optional) Hash encoding.
|
|
14
|
-
* @returns {string} The file checksum.
|
|
15
|
-
*/
|
|
16
|
-
exports.fileChecksum = watt(function* (filePath, options, next) {
|
|
17
|
-
if (!options) {
|
|
18
|
-
options = {};
|
|
19
|
-
}
|
|
20
|
-
if (!options.algorithm) {
|
|
21
|
-
options.algorithm = 'sha1';
|
|
22
|
-
}
|
|
23
|
-
if (!options.encoding) {
|
|
24
|
-
options.encoding = 'hex';
|
|
25
|
-
}
|
|
26
|
-
|
|
27
|
-
const stat = yield fs.stat(filePath, next);
|
|
28
|
-
if (!stat.isFile()) {
|
|
29
|
-
throw new Error(`${filePath} is not a file`);
|
|
30
|
-
}
|
|
31
|
-
|
|
32
|
-
const hash = crypto.createHash(options.algorithm);
|
|
33
|
-
hash.setEncoding(options.encoding);
|
|
34
|
-
|
|
35
|
-
const fileStream = fs.createReadStream(filePath);
|
|
36
|
-
fileStream.pipe(hash, {end: false});
|
|
37
|
-
yield fileStream.once('end', next);
|
|
38
|
-
|
|
39
|
-
hash.end();
|
|
40
|
-
return hash.read();
|
|
41
|
-
});
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const watt = require('gigawatts');
|
|
4
|
+
const crypto = require('crypto');
|
|
5
|
+
const fs = require('fs');
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* Compute the checksum of the file given as parameter using a specified algorithm.
|
|
9
|
+
*
|
|
10
|
+
* @param {string} filePath - The file path.
|
|
11
|
+
* @param {Object} options - Options.
|
|
12
|
+
* @param {string} options.algorithm - For possible values, see https://nodejs.org/api/crypto.html#crypto_crypto_createhash_algorithm_options
|
|
13
|
+
* @param {string} options.encoding - (optional) Hash encoding.
|
|
14
|
+
* @returns {string} The file checksum.
|
|
15
|
+
*/
|
|
16
|
+
exports.fileChecksum = watt(function* (filePath, options, next) {
|
|
17
|
+
if (!options) {
|
|
18
|
+
options = {};
|
|
19
|
+
}
|
|
20
|
+
if (!options.algorithm) {
|
|
21
|
+
options.algorithm = 'sha1';
|
|
22
|
+
}
|
|
23
|
+
if (!options.encoding) {
|
|
24
|
+
options.encoding = 'hex';
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
const stat = yield fs.stat(filePath, next);
|
|
28
|
+
if (!stat.isFile()) {
|
|
29
|
+
throw new Error(`${filePath} is not a file`);
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
const hash = crypto.createHash(options.algorithm);
|
|
33
|
+
hash.setEncoding(options.encoding);
|
|
34
|
+
|
|
35
|
+
const fileStream = fs.createReadStream(filePath);
|
|
36
|
+
fileStream.pipe(hash, {end: false});
|
|
37
|
+
yield fileStream.once('end', next);
|
|
38
|
+
|
|
39
|
+
hash.end();
|
|
40
|
+
return hash.read();
|
|
41
|
+
});
|
package/lib/files.js
CHANGED
|
@@ -1,144 +1,144 @@
|
|
|
1
|
-
'use strict';
|
|
2
|
-
|
|
3
|
-
const path = require('path');
|
|
4
|
-
|
|
5
|
-
const exts = {
|
|
6
|
-
'.aif': 'AIF audio file',
|
|
7
|
-
'.cda': 'CD audio track file',
|
|
8
|
-
'.mid': 'MIDI audio file',
|
|
9
|
-
'.mp3': 'MP3 audio file',
|
|
10
|
-
'.mpa': 'MPEG2 audio file',
|
|
11
|
-
'.ogg': 'Ogg Vorbis audio file',
|
|
12
|
-
'.wav': 'WAV file',
|
|
13
|
-
'.wma': 'WMA audio file',
|
|
14
|
-
'.wpl': 'Windows Media Player playlist',
|
|
15
|
-
'.7z': '7Zip compressed file',
|
|
16
|
-
'.arj': 'ARJ compressed file',
|
|
17
|
-
'.deb': 'Debian software package file',
|
|
18
|
-
'.pkg': 'Package file',
|
|
19
|
-
'.rar': 'RAR file',
|
|
20
|
-
'.rpm': 'Red Hat Package Manager',
|
|
21
|
-
'.tar.gz': 'Tarball compressed file',
|
|
22
|
-
'.z': 'Z compressed file',
|
|
23
|
-
'.zip': 'Zip compressed file',
|
|
24
|
-
'.bin': 'Binary disc image',
|
|
25
|
-
'.dmg': 'macOS X disk image',
|
|
26
|
-
'.iso': 'ISO disc image',
|
|
27
|
-
'.toast': 'Toast disc image',
|
|
28
|
-
'.vcd': 'Virtual CD',
|
|
29
|
-
'.csv': 'Comma separated value file',
|
|
30
|
-
'.dat': 'Data file',
|
|
31
|
-
'.db': 'Database file',
|
|
32
|
-
'.dbf': 'Database file',
|
|
33
|
-
'.log': 'Log file',
|
|
34
|
-
'.mdb': 'Microsoft Access database file',
|
|
35
|
-
'.sav': 'Save file',
|
|
36
|
-
'.sql': 'SQL database file',
|
|
37
|
-
'.tar': 'Linux / Unix tarball file archive',
|
|
38
|
-
'.xml': 'XML file',
|
|
39
|
-
'.apk': 'Android package file',
|
|
40
|
-
'.bat': 'Batch file',
|
|
41
|
-
'.cgi': 'Perl script file',
|
|
42
|
-
'.pl': 'Perl script file',
|
|
43
|
-
'.com': 'MSDOS command file',
|
|
44
|
-
'.exe': 'Executable file',
|
|
45
|
-
'.gadget': 'Windows gadget',
|
|
46
|
-
'.jar': 'Java Archive file',
|
|
47
|
-
'.py': 'Python file',
|
|
48
|
-
'.wsf': 'Windows Script File',
|
|
49
|
-
'.fnt': 'Windows font file',
|
|
50
|
-
'.fon': 'Generic font file',
|
|
51
|
-
'.otf': 'Open type font file',
|
|
52
|
-
'.ttf': 'TrueType font file',
|
|
53
|
-
'.ai': 'Adobe Illustrator file',
|
|
54
|
-
'.bmp': 'Bitmap image',
|
|
55
|
-
'.gif': 'GIF image',
|
|
56
|
-
'.ico': 'Icon file',
|
|
57
|
-
'.jpeg': 'JPEG image',
|
|
58
|
-
'.jpg': 'JPEG image',
|
|
59
|
-
'.png': 'PNG image',
|
|
60
|
-
'.ps': 'PostScript file',
|
|
61
|
-
'.psd': 'PSD image',
|
|
62
|
-
'.svg': 'Scalable Vector Graphics file',
|
|
63
|
-
'.tif': 'TIFF image',
|
|
64
|
-
'.tiff': 'TIFF image',
|
|
65
|
-
'.asp': 'Active Server Page file',
|
|
66
|
-
'.aspx': 'Active Server Page file',
|
|
67
|
-
'.cer': 'Internet security certificate',
|
|
68
|
-
'.cfm': 'ColdFusion Markup file',
|
|
69
|
-
'.css': 'Cascading Style Sheet file',
|
|
70
|
-
'.htm ': 'HTML file',
|
|
71
|
-
'.html': 'HTML file',
|
|
72
|
-
'.js': 'JavaScript file',
|
|
73
|
-
'.jsp': 'Java Server Page file',
|
|
74
|
-
'.part': 'Partially downloaded file',
|
|
75
|
-
'.php': 'PHP file',
|
|
76
|
-
'.rss': 'RSS file',
|
|
77
|
-
'.xhtml': 'XHTML file',
|
|
78
|
-
'.key': 'Keynote presentation',
|
|
79
|
-
'.odp': 'OpenOffice Impress presentation file',
|
|
80
|
-
'.pps': 'PowerPoint slide show',
|
|
81
|
-
'.ppt': 'PowerPoint presentation',
|
|
82
|
-
'.pptx': 'PowerPoint Open XML presentation',
|
|
83
|
-
'.c': 'C and C++ source code file',
|
|
84
|
-
'.class': 'Java class file',
|
|
85
|
-
'.cpp': 'C++ source code file',
|
|
86
|
-
'.cs': 'Visual C# source code file',
|
|
87
|
-
'.h': 'C, C++, and Objective C header file',
|
|
88
|
-
'.java': 'Java Source code file',
|
|
89
|
-
'.sh': 'Bash shell script',
|
|
90
|
-
'.swift': 'Swift source code file',
|
|
91
|
-
'.vb': 'Visual Basic file',
|
|
92
|
-
'.ods': 'OpenOffice Calc spreadsheet file',
|
|
93
|
-
'.xlr': 'Microsoft Works spreadsheet file',
|
|
94
|
-
'.xls': 'Microsoft Excel file',
|
|
95
|
-
'.xlsx': 'Microsoft Excel Open XML spreadsheet file',
|
|
96
|
-
'.bak': 'Backup file',
|
|
97
|
-
'.cab': 'Windows Cabinet file',
|
|
98
|
-
'.cfg': 'Configuration file',
|
|
99
|
-
'.cpl': 'Windows Control panel file',
|
|
100
|
-
'.cur': 'Windows cursor file',
|
|
101
|
-
'.dll': 'DLL file',
|
|
102
|
-
'.dmp': 'Dump file',
|
|
103
|
-
'.drv': 'Device driver file',
|
|
104
|
-
'.icns': 'macOS X icon resource file',
|
|
105
|
-
'.ini': 'Initialization file',
|
|
106
|
-
'.lnk': 'Windows shortcut file',
|
|
107
|
-
'.msi': 'Windows installer package',
|
|
108
|
-
'.sys': 'Windows system file',
|
|
109
|
-
'.tmp': 'Temporary file',
|
|
110
|
-
'.3g2': '3GPP2 multimedia file',
|
|
111
|
-
'.3gp': '3GPP multimedia file',
|
|
112
|
-
'.avi': 'AVI file',
|
|
113
|
-
'.flv': 'Adobe Flash file',
|
|
114
|
-
'.h264': 'H.264 video file',
|
|
115
|
-
'.m4v': 'Apple MP4 video file',
|
|
116
|
-
'.mkv': 'Matroska Multimedia Container',
|
|
117
|
-
'.mov': 'Apple QuickTime movie file',
|
|
118
|
-
'.mp4': 'MPEG4 video file',
|
|
119
|
-
'.mpg': 'MPEG video file',
|
|
120
|
-
'.mpeg': 'MPEG video file',
|
|
121
|
-
'.rm': 'RealMedia file',
|
|
122
|
-
'.swf': 'Shockwave flash file',
|
|
123
|
-
'.vob': 'DVD Video Object',
|
|
124
|
-
'.wmv': 'Windows Media Video file',
|
|
125
|
-
'.doc': 'Microsoft Word file',
|
|
126
|
-
'.docx': 'Microsoft Word file',
|
|
127
|
-
'.odt': 'OpenOffice Writer document file',
|
|
128
|
-
'.pdf': 'PDF file',
|
|
129
|
-
'.rtf': 'Rich Text Format',
|
|
130
|
-
'.tex': 'A LaTeX document file',
|
|
131
|
-
'.txt': 'Plain text file',
|
|
132
|
-
'.wks': 'Microsoft Works file',
|
|
133
|
-
'.wps': 'Microsoft Works file',
|
|
134
|
-
'.wpd': 'WordPerfect document',
|
|
135
|
-
};
|
|
136
|
-
|
|
137
|
-
exports.getFileFilter = function (filePath) {
|
|
138
|
-
const ext = path.extname(filePath);
|
|
139
|
-
const found = exts[ext];
|
|
140
|
-
if (found) {
|
|
141
|
-
return {name: found, extensions: [ext]};
|
|
142
|
-
}
|
|
143
|
-
return {name: '?', extensions: [ext]};
|
|
144
|
-
};
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const path = require('path');
|
|
4
|
+
|
|
5
|
+
const exts = {
|
|
6
|
+
'.aif': 'AIF audio file',
|
|
7
|
+
'.cda': 'CD audio track file',
|
|
8
|
+
'.mid': 'MIDI audio file',
|
|
9
|
+
'.mp3': 'MP3 audio file',
|
|
10
|
+
'.mpa': 'MPEG2 audio file',
|
|
11
|
+
'.ogg': 'Ogg Vorbis audio file',
|
|
12
|
+
'.wav': 'WAV file',
|
|
13
|
+
'.wma': 'WMA audio file',
|
|
14
|
+
'.wpl': 'Windows Media Player playlist',
|
|
15
|
+
'.7z': '7Zip compressed file',
|
|
16
|
+
'.arj': 'ARJ compressed file',
|
|
17
|
+
'.deb': 'Debian software package file',
|
|
18
|
+
'.pkg': 'Package file',
|
|
19
|
+
'.rar': 'RAR file',
|
|
20
|
+
'.rpm': 'Red Hat Package Manager',
|
|
21
|
+
'.tar.gz': 'Tarball compressed file',
|
|
22
|
+
'.z': 'Z compressed file',
|
|
23
|
+
'.zip': 'Zip compressed file',
|
|
24
|
+
'.bin': 'Binary disc image',
|
|
25
|
+
'.dmg': 'macOS X disk image',
|
|
26
|
+
'.iso': 'ISO disc image',
|
|
27
|
+
'.toast': 'Toast disc image',
|
|
28
|
+
'.vcd': 'Virtual CD',
|
|
29
|
+
'.csv': 'Comma separated value file',
|
|
30
|
+
'.dat': 'Data file',
|
|
31
|
+
'.db': 'Database file',
|
|
32
|
+
'.dbf': 'Database file',
|
|
33
|
+
'.log': 'Log file',
|
|
34
|
+
'.mdb': 'Microsoft Access database file',
|
|
35
|
+
'.sav': 'Save file',
|
|
36
|
+
'.sql': 'SQL database file',
|
|
37
|
+
'.tar': 'Linux / Unix tarball file archive',
|
|
38
|
+
'.xml': 'XML file',
|
|
39
|
+
'.apk': 'Android package file',
|
|
40
|
+
'.bat': 'Batch file',
|
|
41
|
+
'.cgi': 'Perl script file',
|
|
42
|
+
'.pl': 'Perl script file',
|
|
43
|
+
'.com': 'MSDOS command file',
|
|
44
|
+
'.exe': 'Executable file',
|
|
45
|
+
'.gadget': 'Windows gadget',
|
|
46
|
+
'.jar': 'Java Archive file',
|
|
47
|
+
'.py': 'Python file',
|
|
48
|
+
'.wsf': 'Windows Script File',
|
|
49
|
+
'.fnt': 'Windows font file',
|
|
50
|
+
'.fon': 'Generic font file',
|
|
51
|
+
'.otf': 'Open type font file',
|
|
52
|
+
'.ttf': 'TrueType font file',
|
|
53
|
+
'.ai': 'Adobe Illustrator file',
|
|
54
|
+
'.bmp': 'Bitmap image',
|
|
55
|
+
'.gif': 'GIF image',
|
|
56
|
+
'.ico': 'Icon file',
|
|
57
|
+
'.jpeg': 'JPEG image',
|
|
58
|
+
'.jpg': 'JPEG image',
|
|
59
|
+
'.png': 'PNG image',
|
|
60
|
+
'.ps': 'PostScript file',
|
|
61
|
+
'.psd': 'PSD image',
|
|
62
|
+
'.svg': 'Scalable Vector Graphics file',
|
|
63
|
+
'.tif': 'TIFF image',
|
|
64
|
+
'.tiff': 'TIFF image',
|
|
65
|
+
'.asp': 'Active Server Page file',
|
|
66
|
+
'.aspx': 'Active Server Page file',
|
|
67
|
+
'.cer': 'Internet security certificate',
|
|
68
|
+
'.cfm': 'ColdFusion Markup file',
|
|
69
|
+
'.css': 'Cascading Style Sheet file',
|
|
70
|
+
'.htm ': 'HTML file',
|
|
71
|
+
'.html': 'HTML file',
|
|
72
|
+
'.js': 'JavaScript file',
|
|
73
|
+
'.jsp': 'Java Server Page file',
|
|
74
|
+
'.part': 'Partially downloaded file',
|
|
75
|
+
'.php': 'PHP file',
|
|
76
|
+
'.rss': 'RSS file',
|
|
77
|
+
'.xhtml': 'XHTML file',
|
|
78
|
+
'.key': 'Keynote presentation',
|
|
79
|
+
'.odp': 'OpenOffice Impress presentation file',
|
|
80
|
+
'.pps': 'PowerPoint slide show',
|
|
81
|
+
'.ppt': 'PowerPoint presentation',
|
|
82
|
+
'.pptx': 'PowerPoint Open XML presentation',
|
|
83
|
+
'.c': 'C and C++ source code file',
|
|
84
|
+
'.class': 'Java class file',
|
|
85
|
+
'.cpp': 'C++ source code file',
|
|
86
|
+
'.cs': 'Visual C# source code file',
|
|
87
|
+
'.h': 'C, C++, and Objective C header file',
|
|
88
|
+
'.java': 'Java Source code file',
|
|
89
|
+
'.sh': 'Bash shell script',
|
|
90
|
+
'.swift': 'Swift source code file',
|
|
91
|
+
'.vb': 'Visual Basic file',
|
|
92
|
+
'.ods': 'OpenOffice Calc spreadsheet file',
|
|
93
|
+
'.xlr': 'Microsoft Works spreadsheet file',
|
|
94
|
+
'.xls': 'Microsoft Excel file',
|
|
95
|
+
'.xlsx': 'Microsoft Excel Open XML spreadsheet file',
|
|
96
|
+
'.bak': 'Backup file',
|
|
97
|
+
'.cab': 'Windows Cabinet file',
|
|
98
|
+
'.cfg': 'Configuration file',
|
|
99
|
+
'.cpl': 'Windows Control panel file',
|
|
100
|
+
'.cur': 'Windows cursor file',
|
|
101
|
+
'.dll': 'DLL file',
|
|
102
|
+
'.dmp': 'Dump file',
|
|
103
|
+
'.drv': 'Device driver file',
|
|
104
|
+
'.icns': 'macOS X icon resource file',
|
|
105
|
+
'.ini': 'Initialization file',
|
|
106
|
+
'.lnk': 'Windows shortcut file',
|
|
107
|
+
'.msi': 'Windows installer package',
|
|
108
|
+
'.sys': 'Windows system file',
|
|
109
|
+
'.tmp': 'Temporary file',
|
|
110
|
+
'.3g2': '3GPP2 multimedia file',
|
|
111
|
+
'.3gp': '3GPP multimedia file',
|
|
112
|
+
'.avi': 'AVI file',
|
|
113
|
+
'.flv': 'Adobe Flash file',
|
|
114
|
+
'.h264': 'H.264 video file',
|
|
115
|
+
'.m4v': 'Apple MP4 video file',
|
|
116
|
+
'.mkv': 'Matroska Multimedia Container',
|
|
117
|
+
'.mov': 'Apple QuickTime movie file',
|
|
118
|
+
'.mp4': 'MPEG4 video file',
|
|
119
|
+
'.mpg': 'MPEG video file',
|
|
120
|
+
'.mpeg': 'MPEG video file',
|
|
121
|
+
'.rm': 'RealMedia file',
|
|
122
|
+
'.swf': 'Shockwave flash file',
|
|
123
|
+
'.vob': 'DVD Video Object',
|
|
124
|
+
'.wmv': 'Windows Media Video file',
|
|
125
|
+
'.doc': 'Microsoft Word file',
|
|
126
|
+
'.docx': 'Microsoft Word file',
|
|
127
|
+
'.odt': 'OpenOffice Writer document file',
|
|
128
|
+
'.pdf': 'PDF file',
|
|
129
|
+
'.rtf': 'Rich Text Format',
|
|
130
|
+
'.tex': 'A LaTeX document file',
|
|
131
|
+
'.txt': 'Plain text file',
|
|
132
|
+
'.wks': 'Microsoft Works file',
|
|
133
|
+
'.wps': 'Microsoft Works file',
|
|
134
|
+
'.wpd': 'WordPerfect document',
|
|
135
|
+
};
|
|
136
|
+
|
|
137
|
+
exports.getFileFilter = function (filePath) {
|
|
138
|
+
const ext = path.extname(filePath);
|
|
139
|
+
const found = exts[ext];
|
|
140
|
+
if (found) {
|
|
141
|
+
return {name: found, extensions: [ext]};
|
|
142
|
+
}
|
|
143
|
+
return {name: '?', extensions: [ext]};
|
|
144
|
+
};
|
package/lib/job-queue.js
CHANGED
|
@@ -1,114 +1,113 @@
|
|
|
1
|
-
'use strict';
|
|
2
|
-
|
|
3
|
-
const moduleName = 'job-queue';
|
|
4
|
-
const clc = require('cli-color');
|
|
5
|
-
const watt = require('gigawatts');
|
|
6
|
-
const {locks} = require('xcraft-core-utils');
|
|
7
|
-
const runner = require('./runnerInstance.js');
|
|
8
|
-
const throttle = require('lodash/throttle');
|
|
9
|
-
const defaultOptions = {
|
|
10
|
-
priorityGroup: 'default',
|
|
11
|
-
waitOn: [], //groups to wait
|
|
12
|
-
waitDelay: 50,
|
|
13
|
-
maxAttempt: 100,
|
|
14
|
-
useLogger: false,
|
|
15
|
-
};
|
|
16
|
-
class JobQueue {
|
|
17
|
-
constructor(name, runner, parallelLimit, options) {
|
|
18
|
-
//override default options
|
|
19
|
-
if (!options) {
|
|
20
|
-
options = defaultOptions;
|
|
21
|
-
} else {
|
|
22
|
-
options = {...defaultOptions, ...options};
|
|
23
|
-
}
|
|
24
|
-
|
|
25
|
-
this.log =
|
|
26
|
-
options.useLogger && require('xcraft-core-log')(`${moduleName}`, null);
|
|
27
|
-
this.name = name;
|
|
28
|
-
this.priorityGroup = options.priorityGroup;
|
|
29
|
-
this.waitOn = options.waitOn;
|
|
30
|
-
this.maxAttempt = options.maxAttempt;
|
|
31
|
-
this.attempt = 0;
|
|
32
|
-
this.waitDelay = options.waitDelay;
|
|
33
|
-
this.channel = name.toLowerCase().replace(/\.|\[|\//g, '-');
|
|
34
|
-
this.runner = watt(function* (...args) {
|
|
35
|
-
this.
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
this.
|
|
40
|
-
this.
|
|
41
|
-
this.
|
|
42
|
-
this.
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
this.
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
module.exports = JobQueue;
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const moduleName = 'job-queue';
|
|
4
|
+
const clc = require('cli-color');
|
|
5
|
+
const watt = require('gigawatts');
|
|
6
|
+
const {locks} = require('xcraft-core-utils');
|
|
7
|
+
const runner = require('./runnerInstance.js');
|
|
8
|
+
const throttle = require('lodash/throttle');
|
|
9
|
+
const defaultOptions = {
|
|
10
|
+
priorityGroup: 'default',
|
|
11
|
+
waitOn: [], //groups to wait
|
|
12
|
+
waitDelay: 50,
|
|
13
|
+
maxAttempt: 100,
|
|
14
|
+
useLogger: false,
|
|
15
|
+
};
|
|
16
|
+
class JobQueue {
|
|
17
|
+
constructor(name, runner, parallelLimit, options) {
|
|
18
|
+
//override default options
|
|
19
|
+
if (!options) {
|
|
20
|
+
options = defaultOptions;
|
|
21
|
+
} else {
|
|
22
|
+
options = {...defaultOptions, ...options};
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
this.log =
|
|
26
|
+
options.useLogger && require('xcraft-core-log')(`${moduleName}`, null);
|
|
27
|
+
this.name = name;
|
|
28
|
+
this.priorityGroup = options.priorityGroup;
|
|
29
|
+
this.waitOn = options.waitOn;
|
|
30
|
+
this.maxAttempt = options.maxAttempt;
|
|
31
|
+
this.attempt = 0;
|
|
32
|
+
this.waitDelay = options.waitDelay;
|
|
33
|
+
this.channel = name.toLowerCase().replace(/\.|\[|\//g, '-');
|
|
34
|
+
this.runner = watt(function* (...args) {
|
|
35
|
+
this.runningSamples++;
|
|
36
|
+
yield* runner(...args);
|
|
37
|
+
});
|
|
38
|
+
this.parallelLimit = parallelLimit;
|
|
39
|
+
this.waiting = new Map();
|
|
40
|
+
this.running = 0;
|
|
41
|
+
this.runningSamples = 0;
|
|
42
|
+
this._mutex = new locks.Mutex();
|
|
43
|
+
const busClient = require('xcraft-core-busclient').getGlobal();
|
|
44
|
+
this.resp = busClient.newResponse('job-queue', 'token');
|
|
45
|
+
watt.wrapAll(this);
|
|
46
|
+
this.notify = throttle(this._notify, 500).bind(this);
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
get maxAttemptReached() {
|
|
50
|
+
return this.attempt >= this.maxAttempt;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
_notify(runningSamples) {
|
|
54
|
+
this.resp.events.send('<job-queue.sampled>', {
|
|
55
|
+
channel: this.channel,
|
|
56
|
+
sample: runningSamples,
|
|
57
|
+
current: 0,
|
|
58
|
+
total: 0,
|
|
59
|
+
});
|
|
60
|
+
this.runningSamples = 0;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
_log() {
|
|
64
|
+
if (!this.log) {
|
|
65
|
+
return;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
const msg = `«${clc.blackBright.bold(this.name)}» waiting:${
|
|
69
|
+
this.waiting.size
|
|
70
|
+
} running:${this.running}`;
|
|
71
|
+
this.log.info(msg);
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
_dbg(debugMessage) {
|
|
75
|
+
if (!this.log) {
|
|
76
|
+
return;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
const msg = `«${clc.blackBright.bold(this.name)}» ${debugMessage}`;
|
|
80
|
+
this.log.dbg(msg);
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
err(ex) {
|
|
84
|
+
const msg = `«${clc.blackBright.bold(this.name)}» ${
|
|
85
|
+
ex.stack || ex.message || ex
|
|
86
|
+
}`;
|
|
87
|
+
|
|
88
|
+
if (!this.log) {
|
|
89
|
+
console.error(msg);
|
|
90
|
+
return;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
this.log.err(msg);
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
push(job) {
|
|
97
|
+
try {
|
|
98
|
+
this.waiting.set(job.id, job);
|
|
99
|
+
} finally {
|
|
100
|
+
//launch macro-task
|
|
101
|
+
setTimeout(runner.instance.run, 0, this);
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
dispose() {
|
|
106
|
+
this.notify(0);
|
|
107
|
+
this.resp.events.send('<job-queue.disposed>', {
|
|
108
|
+
channel: this.channel,
|
|
109
|
+
});
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
module.exports = JobQueue;
|
package/lib/js.js
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
|
-
'use strict';
|
|
2
|
-
|
|
3
|
-
exports.isFunction = function (fn) {
|
|
4
|
-
return typeof fn === 'function';
|
|
5
|
-
};
|
|
6
|
-
|
|
7
|
-
exports.isGenerator = function (fn) {
|
|
8
|
-
return (
|
|
9
|
-
fn &&
|
|
10
|
-
exports.isFunction(fn) &&
|
|
11
|
-
fn.constructor &&
|
|
12
|
-
fn.constructor.name === 'GeneratorFunction'
|
|
13
|
-
);
|
|
14
|
-
};
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
exports.isFunction = function (fn) {
|
|
4
|
+
return typeof fn === 'function';
|
|
5
|
+
};
|
|
6
|
+
|
|
7
|
+
exports.isGenerator = function (fn) {
|
|
8
|
+
return (
|
|
9
|
+
fn &&
|
|
10
|
+
exports.isFunction(fn) &&
|
|
11
|
+
fn.constructor &&
|
|
12
|
+
fn.constructor.name === 'GeneratorFunction'
|
|
13
|
+
);
|
|
14
|
+
};
|
package/lib/json.js
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
|
-
'use strict';
|
|
2
|
-
|
|
3
|
-
var fs = require('fs');
|
|
4
|
-
|
|
5
|
-
exports.fromFile = function (jsonFile) {
|
|
6
|
-
var data = fs.readFileSync(jsonFile, 'utf8');
|
|
7
|
-
return JSON.parse(data);
|
|
8
|
-
};
|
|
9
|
-
|
|
10
|
-
exports.toFile = function (json, destFile) {
|
|
11
|
-
fs.writeFileSync(destFile, JSON.stringify(json), 'utf8');
|
|
12
|
-
};
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
var fs = require('fs');
|
|
4
|
+
|
|
5
|
+
exports.fromFile = function (jsonFile) {
|
|
6
|
+
var data = fs.readFileSync(jsonFile, 'utf8');
|
|
7
|
+
return JSON.parse(data);
|
|
8
|
+
};
|
|
9
|
+
|
|
10
|
+
exports.toFile = function (json, destFile) {
|
|
11
|
+
fs.writeFileSync(destFile, JSON.stringify(json), 'utf8');
|
|
12
|
+
};
|