total5 0.0.8-1 → 0.0.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/builders.js +19 -6
- package/changelog.txt +5 -0
- package/htmlparser.js +95 -85
- package/index.js +1 -1
- package/package.json +1 -1
- package/utils.js +1 -0
package/builders.js
CHANGED
|
@@ -47,6 +47,11 @@ Options.prototype = {
|
|
|
47
47
|
this.payload = value;
|
|
48
48
|
},
|
|
49
49
|
|
|
50
|
+
get hostname() {
|
|
51
|
+
let ctrl = this.controller;
|
|
52
|
+
return ctrl ? ((ctrl.protocol || 'https') + '://' + ctrl.headers.host) : null;
|
|
53
|
+
},
|
|
54
|
+
|
|
50
55
|
get url() {
|
|
51
56
|
return (this.controller ? this.controller.url : '') || '';
|
|
52
57
|
},
|
|
@@ -121,6 +126,7 @@ Options.prototype.promisify = function(fn, a, b, c) {
|
|
|
121
126
|
|
|
122
127
|
var callback = function(err, response) {
|
|
123
128
|
if (err)
|
|
129
|
+
|
|
124
130
|
$.invalid(err);
|
|
125
131
|
else
|
|
126
132
|
resolve(response);
|
|
@@ -941,9 +947,12 @@ function restbuilder_callback(err, response) {
|
|
|
941
947
|
|
|
942
948
|
if (self.options.custom) {
|
|
943
949
|
if (self.$resolve) {
|
|
944
|
-
if (err)
|
|
945
|
-
self
|
|
946
|
-
|
|
950
|
+
if (err) {
|
|
951
|
+
if (self.$)
|
|
952
|
+
self.$.invalid(err);
|
|
953
|
+
else
|
|
954
|
+
self.$reject(err);
|
|
955
|
+
} else
|
|
947
956
|
self.$resolve(response);
|
|
948
957
|
self.$ = null;
|
|
949
958
|
self.$reject = null;
|
|
@@ -1042,8 +1051,12 @@ function restbuilder_callback(err, response) {
|
|
|
1042
1051
|
|
|
1043
1052
|
if (self.$resolve) {
|
|
1044
1053
|
|
|
1045
|
-
if (err)
|
|
1046
|
-
self
|
|
1054
|
+
if (err) {
|
|
1055
|
+
if (self.$)
|
|
1056
|
+
self.$.invalid(err);
|
|
1057
|
+
else
|
|
1058
|
+
self.$reject(err);
|
|
1059
|
+
}
|
|
1047
1060
|
else
|
|
1048
1061
|
self.$resolve(val);
|
|
1049
1062
|
|
|
@@ -1339,7 +1352,7 @@ ActionCaller.prototype.exec = function() {
|
|
|
1339
1352
|
}
|
|
1340
1353
|
}
|
|
1341
1354
|
|
|
1342
|
-
if (action.sa) {
|
|
1355
|
+
if (action.sa || action.su) {
|
|
1343
1356
|
if (!$.user || (!$.user.sa && !$.user.su)) {
|
|
1344
1357
|
$.invalid(401);
|
|
1345
1358
|
return;
|
package/changelog.txt
CHANGED
|
@@ -10,6 +10,11 @@
|
|
|
10
10
|
- added a better support for `*:tagname` prefixes in HTMLParser
|
|
11
11
|
- fixed context in the components
|
|
12
12
|
- fixed URL downloading templates in `TEMPLATE()` method
|
|
13
|
+
- fixed parsing of unpaired elements in HTMLParser
|
|
14
|
+
- added support for `log` extension in static file processing
|
|
15
|
+
- fixed selectors in the HTMLParser
|
|
16
|
+
- added `$.hostname {String}` to the `SchemaOptions`
|
|
17
|
+
- fixed `RESTBuilder.promise()`
|
|
13
18
|
|
|
14
19
|
========================
|
|
15
20
|
0.0.7
|
package/htmlparser.js
CHANGED
|
@@ -24,23 +24,23 @@ function parseRule(selector, output) {
|
|
|
24
24
|
rule.attrs = [];
|
|
25
25
|
rule.output = output || [];
|
|
26
26
|
|
|
27
|
+
// > div
|
|
27
28
|
// div div[name="Peter Sirka"]
|
|
28
29
|
// div > div[name="Peter Sirka"]
|
|
29
30
|
|
|
30
|
-
// for (var c of selector) {
|
|
31
|
-
// if (c === '>')
|
|
32
|
-
// console.log(c);
|
|
33
|
-
// }
|
|
34
|
-
|
|
35
31
|
var cache = [];
|
|
36
32
|
|
|
37
|
-
selector = selector.replace(/\[.*?\]/gi, text => '[' + (cache.push(text) - 1) + ']').replace(/(\s)?>(\s)?/, '>').replace(/\s{2}/g, '');
|
|
33
|
+
selector = selector.replace(/\[.*?\]/gi, text => '[' + (cache.push(text) - 1) + ']').replace(/(\s)?>(\s)?/, '>').replace(/\s{2}/g, '').trim();
|
|
34
|
+
|
|
35
|
+
rule.notravelse = selector[0] === '>';
|
|
36
|
+
|
|
37
|
+
if (rule.notravelse)
|
|
38
|
+
selector = selector.substring(1);
|
|
38
39
|
|
|
39
40
|
var m = selector.match(/>|\s/);
|
|
40
41
|
if (m) {
|
|
41
|
-
var nested = selector.substring(m.index
|
|
42
|
+
var nested = selector.substring(m.index).trim().replace(/\[\d+\]/g, text => cache[+text.substring(1, text.length - 1)]);
|
|
42
43
|
rule.nested = parseRule(nested, rule.output);
|
|
43
|
-
rule.direct = m[0] === '>';
|
|
44
44
|
selector = selector.substring(0, m.index).trim();
|
|
45
45
|
}
|
|
46
46
|
|
|
@@ -156,6 +156,48 @@ function extendarr(output) {
|
|
|
156
156
|
return output;
|
|
157
157
|
}
|
|
158
158
|
|
|
159
|
+
function compare(rule, node) {
|
|
160
|
+
|
|
161
|
+
if (rule.prefix === '*') {
|
|
162
|
+
var tagName = node.tagName;
|
|
163
|
+
if (node.prefix)
|
|
164
|
+
tagName = tagName.substring(node.prefix.length);
|
|
165
|
+
if (tagName !== rule.tagName)
|
|
166
|
+
return false;
|
|
167
|
+
} else {
|
|
168
|
+
if (rule.tagName && rule.tagName !== node.tagName)
|
|
169
|
+
return false;
|
|
170
|
+
if (rule.prefix && rule.prefix !== node.prefix)
|
|
171
|
+
return false;
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
if (rule.attrs.length) {
|
|
175
|
+
for (var attr of rule.attrs) {
|
|
176
|
+
switch (attr.id) {
|
|
177
|
+
case 'class':
|
|
178
|
+
var tmp = node.attrs[attr.id];
|
|
179
|
+
if (tmp) {
|
|
180
|
+
tmp = tmp.split(' ');
|
|
181
|
+
if (!tmp.includes(attr.value))
|
|
182
|
+
return false;
|
|
183
|
+
} else
|
|
184
|
+
return false;
|
|
185
|
+
break;
|
|
186
|
+
default:
|
|
187
|
+
if (attr.value) {
|
|
188
|
+
if (node.attrs[attr.id] !== attr.value)
|
|
189
|
+
return false;
|
|
190
|
+
} else if (node.attrs[attr.id] === undefined)
|
|
191
|
+
return false;
|
|
192
|
+
|
|
193
|
+
break;
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
return true;
|
|
199
|
+
}
|
|
200
|
+
|
|
159
201
|
HTMLElement.prototype.find = function(selector, reverse) {
|
|
160
202
|
|
|
161
203
|
var self = this;
|
|
@@ -166,81 +208,33 @@ HTMLElement.prototype.find = function(selector, reverse) {
|
|
|
166
208
|
for (var sel of selectors)
|
|
167
209
|
rules.push(parseRule(sel.trim()));
|
|
168
210
|
|
|
169
|
-
var browse = function(rule, children
|
|
211
|
+
var browse = function(rule, children) {
|
|
170
212
|
|
|
171
|
-
for (
|
|
213
|
+
for (let node of children) {
|
|
172
214
|
|
|
173
215
|
if (!node.tagName)
|
|
174
216
|
continue;
|
|
175
217
|
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
if (rule.prefix === '*') {
|
|
179
|
-
|
|
180
|
-
var tagName = node.tagName;
|
|
181
|
-
if (node.prefix)
|
|
182
|
-
tagName = tagName.substring(node.prefix.length);
|
|
183
|
-
|
|
184
|
-
if (tagName !== rule.tagName)
|
|
185
|
-
skip = true;
|
|
186
|
-
|
|
187
|
-
} else {
|
|
188
|
-
if (rule.tagName && rule.tagName !== node.tagName)
|
|
189
|
-
skip = true;
|
|
190
|
-
if (rule.prefix && rule.prefix !== node.prefix)
|
|
191
|
-
skip = true;
|
|
192
|
-
}
|
|
193
|
-
|
|
194
|
-
if (rule.attrs.length && !skip) {
|
|
195
|
-
for (var attr of rule.attrs) {
|
|
196
|
-
switch (attr.id) {
|
|
197
|
-
|
|
198
|
-
case 'class':
|
|
199
|
-
var tmp = node.attrs[attr.id];
|
|
200
|
-
if (tmp) {
|
|
201
|
-
tmp = tmp.split(' ');
|
|
202
|
-
if (!tmp.includes(attr.value))
|
|
203
|
-
skip = true;
|
|
204
|
-
} else
|
|
205
|
-
skip = true;
|
|
206
|
-
break;
|
|
207
|
-
|
|
208
|
-
default:
|
|
209
|
-
if (attr.value) {
|
|
210
|
-
if (node.attrs[attr.id] !== attr.value)
|
|
211
|
-
skip = true;
|
|
212
|
-
} else if (node.attrs[attr.id] === undefined)
|
|
213
|
-
skip = true;
|
|
214
|
-
|
|
215
|
-
break;
|
|
216
|
-
}
|
|
218
|
+
let children = reverse ? [node.parentNode] : node.children;
|
|
217
219
|
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
220
|
+
if (!compare(rule, node)) {
|
|
221
|
+
if (!rule.notravelse)
|
|
222
|
+
browse(rule, children);
|
|
223
|
+
continue;
|
|
221
224
|
}
|
|
222
225
|
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
if (skip && parent.direct)
|
|
227
|
-
continue;
|
|
228
|
-
}
|
|
226
|
+
// types complexType attribute
|
|
227
|
+
// types > complexType attribute
|
|
228
|
+
// types complexType > attribute
|
|
229
229
|
|
|
230
230
|
if (rule.nested) {
|
|
231
|
+
browse(rule.nested, children);
|
|
232
|
+
} else {
|
|
231
233
|
|
|
232
|
-
|
|
233
|
-
browse(rule.nested, reverse ? [node.parentNode] : node.children, rule);
|
|
234
|
-
|
|
235
|
-
// Again same
|
|
236
|
-
if (!parent)
|
|
237
|
-
browse(rule, reverse ? [node.parentNode] : node.children);
|
|
234
|
+
rule.output.push(node);
|
|
238
235
|
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
rule.output.push(node);
|
|
242
|
-
if (next)
|
|
243
|
-
browse(rule, reverse ? [node.parentNode] : node.children);
|
|
236
|
+
if (!rule.notravelse)
|
|
237
|
+
browse(rule, children);
|
|
244
238
|
}
|
|
245
239
|
}
|
|
246
240
|
};
|
|
@@ -249,7 +243,9 @@ HTMLElement.prototype.find = function(selector, reverse) {
|
|
|
249
243
|
return output;
|
|
250
244
|
|
|
251
245
|
for (var rule of rules) {
|
|
246
|
+
|
|
252
247
|
browse(rule, reverse ? [self.parentNode] : self.children);
|
|
248
|
+
|
|
253
249
|
if (rule.output.length)
|
|
254
250
|
output.push.apply(output, rule.output);
|
|
255
251
|
}
|
|
@@ -550,7 +546,6 @@ function parseHTML(html, trim, onerror, isxml) {
|
|
|
550
546
|
return '';
|
|
551
547
|
}
|
|
552
548
|
|
|
553
|
-
|
|
554
549
|
if (beg > 0) {
|
|
555
550
|
tmp = str.substring(0, beg);
|
|
556
551
|
|
|
@@ -617,12 +612,6 @@ function parseHTML(html, trim, onerror, isxml) {
|
|
|
617
612
|
return str;
|
|
618
613
|
} else {
|
|
619
614
|
switch (dom.tagName) {
|
|
620
|
-
case 'BR':
|
|
621
|
-
case 'HR':
|
|
622
|
-
case 'IMG':
|
|
623
|
-
case 'INPUT':
|
|
624
|
-
case 'META':
|
|
625
|
-
case 'LINK':
|
|
626
615
|
case 'SOURCE':
|
|
627
616
|
case 'AREA':
|
|
628
617
|
case 'COL':
|
|
@@ -633,17 +622,41 @@ function parseHTML(html, trim, onerror, isxml) {
|
|
|
633
622
|
}
|
|
634
623
|
}
|
|
635
624
|
|
|
636
|
-
var pos =
|
|
625
|
+
var pos = -1;
|
|
637
626
|
var tagBeg = '<' + dom.raw;
|
|
638
627
|
var tagEnd = '</' + dom.raw + '>';
|
|
628
|
+
var unpair = false;
|
|
639
629
|
|
|
640
630
|
while (true) {
|
|
641
631
|
|
|
642
632
|
if (counter++ > 10000)
|
|
643
633
|
break;
|
|
644
634
|
|
|
635
|
+
if (unpair)
|
|
636
|
+
unpair = false;
|
|
637
|
+
else
|
|
638
|
+
end = str.indexOf(tagEnd, pos);
|
|
639
|
+
|
|
640
|
+
// Tries to find the same tag
|
|
645
641
|
beg = str.indexOf(tagBeg, pos);
|
|
646
|
-
|
|
642
|
+
if (beg !== -1) {
|
|
643
|
+
|
|
644
|
+
// another one with the same type
|
|
645
|
+
// check unpair
|
|
646
|
+
|
|
647
|
+
let posend = str.indexOf('>', beg + tagBeg.length);
|
|
648
|
+
if (posend === -1) {
|
|
649
|
+
// tag is not closed
|
|
650
|
+
return '';
|
|
651
|
+
}
|
|
652
|
+
|
|
653
|
+
if (str[posend - 1] === '/') {
|
|
654
|
+
// unpair
|
|
655
|
+
pos = posend + 1;
|
|
656
|
+
unpair = true;
|
|
657
|
+
continue;
|
|
658
|
+
}
|
|
659
|
+
}
|
|
647
660
|
|
|
648
661
|
// Fallback for the non-exists end tag
|
|
649
662
|
if (end === -1) {
|
|
@@ -658,20 +671,18 @@ function parseHTML(html, trim, onerror, isxml) {
|
|
|
658
671
|
}
|
|
659
672
|
|
|
660
673
|
if (beg === -1 || end < beg) {
|
|
661
|
-
|
|
662
674
|
pos = end + tagEnd.length;
|
|
663
|
-
|
|
664
675
|
if (count) {
|
|
665
676
|
count--;
|
|
666
677
|
continue;
|
|
667
678
|
}
|
|
668
|
-
|
|
669
679
|
break;
|
|
670
680
|
}
|
|
681
|
+
|
|
671
682
|
}
|
|
672
683
|
|
|
673
684
|
var inner = str.substring(0, pos - tagEnd.length);
|
|
674
|
-
if (inner.indexOf('<') === -1 || (/\<(script|style|template)/).test(tag)) {
|
|
685
|
+
if (inner.indexOf('<') === -1 || (!isxml && (/\<(script|style|template)/).test(tag))) {
|
|
675
686
|
if (trim)
|
|
676
687
|
inner = inner.trim();
|
|
677
688
|
if (inner)
|
|
@@ -686,7 +697,6 @@ function parseHTML(html, trim, onerror, isxml) {
|
|
|
686
697
|
if (str && str.indexOf('<') === -1) {
|
|
687
698
|
if (trim)
|
|
688
699
|
str = str.trim();
|
|
689
|
-
|
|
690
700
|
// Commented because it inserts the same textContent twice
|
|
691
701
|
// str && parent.children.push(makeText(parent, str));
|
|
692
702
|
}
|
package/index.js
CHANGED
|
@@ -354,7 +354,7 @@ function unlink(arr, callback) {
|
|
|
354
354
|
CONF.$httpmaxsize = 256; // kB
|
|
355
355
|
CONF.$httprangebuffer = 5120; // 5 MB
|
|
356
356
|
CONF.$httptimeout = 5; // 5 seconds
|
|
357
|
-
CONF.$httpfiles = { flac: true, jpg: true, jpeg: true, png: true, gif: true, ico: true, wasm: true, js: true, mjs: true, css: true, txt: true, xml: true, woff: true, woff2: true, otf: true, ttf: true, eot: true, svg: true, zip: true, rar: true, pdf: true, docx: true, xlsx: true, doc: true, xls: true, html: true, htm: true, appcache: true, manifest: true, map: true, ogv: true, ogg: true, mp4: true, mp3: true, webp: true, webm: true, swf: true, package: true, json: true, ui: true, md: true, m4v: true, jsx: true, heif: true, heic: true, ics: true, ts: true, m3u8: true, wav: true, xsd: true, xsl: true, xslt: true, ipynb: true, ijsnb: true };
|
|
357
|
+
CONF.$httpfiles = { flac: true, jpg: true, jpeg: true, png: true, gif: true, ico: true, wasm: true, js: true, mjs: true, css: true, txt: true, xml: true, woff: true, woff2: true, otf: true, ttf: true, eot: true, svg: true, zip: true, rar: true, pdf: true, docx: true, xlsx: true, doc: true, xls: true, html: true, htm: true, appcache: true, manifest: true, map: true, ogv: true, ogg: true, mp4: true, mp3: true, webp: true, webm: true, swf: true, package: true, json: true, ui: true, md: true, m4v: true, jsx: true, heif: true, heic: true, ics: true, ts: true, m3u8: true, wav: true, xsd: true, xsl: true, xslt: true, ipynb: true, ijsnb: true, log: true };
|
|
358
358
|
CONF.$httpchecktypes = true; // for multipart data only
|
|
359
359
|
CONF.$httpmaxage = 60; // in seconds
|
|
360
360
|
CONF.$httpmaxkeys = 33;
|
package/package.json
CHANGED