xml-twig 1.3.10 → 1.3.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (2) hide show
  1. package/package.json +1 -1
  2. package/samples/memory-test.js +45 -41
package/package.json CHANGED
@@ -5,7 +5,7 @@
5
5
  },
6
6
  "name": "xml-twig",
7
7
  "description": "Node module for processing huge XML documents in tree mode",
8
- "version": "1.3.10",
8
+ "version": "1.3.11",
9
9
  "main": "twig.js",
10
10
  "directories": {
11
11
  "doc": "doc"
package/samples/memory-test.js CHANGED
@@ -2,26 +2,26 @@ const fs = require('fs');
2
2
  const process = require('process');
3
3
  const twig = require('xml-twig');
4
4
 
5
- let NE = 0;
6
- console.log('Starting...')
7
- let parser = twig.createParser([{ tag: 'subsession', function: anyHandler }], { method: 'expat' })
8
- let reader = fs.createReadStream(`${__dirname}/20231019015552.1-MSRAN.xml`);
5
+ let Entry = 0;
6
+
7
+ let parser = twig.createParser([{ tag: 'Entry', function: EntryHandler }], { method: 'expat' })
8
+ // http://aiweb.cs.washington.edu/research/projects/xmltk/xmldata/data/SwissProt/SwissProt.xml.gz
9
+ let reader = fs.createReadStream(`SwissProt.xml`);
9
10
  reader.pipe(parser);
10
11
 
11
- function anyHandler(elt) {
12
- NE++;
13
- if (NE % 5 === 0) {
12
+ function EntryHandler(elt) {
13
+ Entry++;
14
+ if (Entry % 10000 === 0) {
15
+ console.log(`Entry ${Entry}`)
16
+ let len = elt.root().writer().toString().length;
17
+ len = Math.round((len / 1024 / 1024 + Number.EPSILON) * 100) / 100;
18
+ console.log(`Size of XML string: ${len} MiB`);
14
19
  for (const [key, value] of Object.entries(process.memoryUsage())) {
15
20
  console.log(` Memory usage by ${key}, ${Math.round((value / 1024 / 1024 + Number.EPSILON) * 100) / 100} MiB`)
16
21
  }
17
- }
18
- //elt.purge();
22
+ }
19
23
  }
20
24
 
21
- reader.on('end', () => {
22
- console.log(`All done`);
23
- });
24
-
25
25
 
26
26
 
27
27
  /*
@@ -29,44 +29,48 @@ reader.on('end', () => {
29
29
  * Results
30
30
  **********************
31
31
 
32
+ # Set memory limit to 4GiB for demonstration reasons:
32
33
  NODE_OPTIONS=--max-old-space-size=4096
33
34
 
34
- 5 NE
35
- Memory usage by rss, 1070.65 MiB
36
- Memory usage by heapTotal, 1025.37 MiB
37
- Memory usage by heapUsed, 989.72 MiB
38
- Memory usage by external, 1.24 MiB
39
- Memory usage by arrayBuffers, 0.83 MiB
40
- 10 NE
41
- Memory usage by rss, 2816.7 MiB
42
- Memory usage by heapTotal, 2760.62 MiB
43
- Memory usage by heapUsed, 2690.95 MiB
44
- Memory usage by external, 1.42 MiB
45
- Memory usage by arrayBuffers, 1.02 MiB
35
+ Entry 10000
36
+ Size of XML string: 21.08 MiB
37
+ Memory usage by rss, 1897.36 MiB
38
+ Memory usage by heapTotal, 1855.68 MiB
39
+ Memory usage by heapUsed, 1807.43 MiB
40
+ Memory usage by external, 1.13 MiB
41
+ Memory usage by arrayBuffers, 0.7 MiB
42
+
43
+ Entry 20000
44
+ Size of XML string: 40.86 MiB
45
+ Memory usage by rss, 3615.75 MiB
46
+ Memory usage by heapTotal, 3562.62 MiB
47
+ Memory usage by heapUsed, 3482.97 MiB
48
+ Memory usage by external, 0.63 MiB
49
+ Memory usage by arrayBuffers, 0.2 MiB
46
50
 
47
51
  <--- Last few GCs --->
48
52
 
49
- [6508:00000223570D04D0] 20692 ms: Scavenge 4034.0 (4122.1) -> 4032.4 (4135.1) MB, 13.7 / 0.0 ms (average mu = 0.431, current mu = 0.348) allocation failure;
50
- [6508:00000223570D04D0] 23749 ms: Mark-sweep 4046.7 (4135.1) -> 4044.6 (4150.4) MB, 3030.4 / 0.0 ms (average mu = 0.250, current mu = 0.055) allocation failure; scavenge might not succeed
53
+ [18648:000001F930FC8F90] 13906 ms: Scavenge 4047.7 (4123.2) -> 4047.3 (4133.7) MB, 3.4 / 0.0 ms (average mu = 0.332, current mu = 0.185) allocation failure;
54
+ [18648:000001F930FC8F90] 13916 ms: Scavenge 4054.1 (4133.7) -> 4053.8 (4135.2) MB, 4.0 / 0.0 ms (average mu = 0.332, current mu = 0.185) allocation failure;
55
+ [18648:000001F930FC8F90] 13925 ms: Scavenge 4055.1 (4135.2) -> 4054.3 (4156.2) MB, 8.3 / 0.0 ms (average mu = 0.332, current mu = 0.185) allocation failure;
51
56
 
52
57
 
53
58
  <--- JS stacktrace --->
54
59
 
55
60
  FATAL ERROR: Reached heap limit Allocation failed - JavaScript heap out of memory
56
- 1: 00007FF7BCDB9E7F node_api_throw_syntax_error+175967
57
- 2: 00007FF7BCD40C06 SSL_get_quiet_shutdown+65750
58
- 3: 00007FF7BCD41FC2 SSL_get_quiet_shutdown+70802
59
- 4: 00007FF7BD7DA214 v8::Isolate::ReportExternalAllocationLimitReached+116
60
- 5: 00007FF7BD7C5572 v8::Isolate::Exit+674
61
- 6: 00007FF7BD6473CC v8::internal::EmbedderStackStateScope::ExplicitScopeForTesting+124
62
- 7: 00007FF7BD6445EB v8::internal::Heap::CollectGarbage+3963
63
- 8: 00007FF7BD65A823 v8::internal::HeapAllocator::AllocateRawWithLightRetrySlowPath+2099
64
- 9: 00007FF7BD65B0CD v8::internal::HeapAllocator::AllocateRawWithRetryOrFailSlowPath+93
65
- 10: 00007FF7BD66A903 v8::internal::Factory::NewFillerObject+851
66
- 11: 00007FF7BD35BEB5 v8::internal::DateCache::Weekday+1349
67
- 12: 00007FF7BD8778B1 v8::internal::SetupIsolateDelegate::SetupHeap+558193
68
- 13: 00007FF73D9F10A1
69
-
61
+ 1: 00007FF604481C7F node_api_throw_syntax_error+175855
62
+ 2: 00007FF604406476 EVP_MD_meth_get_input_blocksize+59654
63
+ 3: 00007FF604408160 EVP_MD_meth_get_input_blocksize+67056
64
+ 4: 00007FF604EB0434 v8::Isolate::ReportExternalAllocationLimitReached+116
65
+ 5: 00007FF604E9B7C2 v8::Isolate::Exit+674
66
+ 6: 00007FF604D1D65C v8::internal::EmbedderStackStateScope::ExplicitScopeForTesting+124
67
+ 7: 00007FF604D1A87B v8::internal::Heap::CollectGarbage+3963
68
+ 8: 00007FF604D30AB3 v8::internal::HeapAllocator::AllocateRawWithLightRetrySlowPath+2099
69
+ 9: 00007FF604D3135D v8::internal::HeapAllocator::AllocateRawWithRetryOrFailSlowPath+93
70
+ 10: 00007FF604D40B20 v8::internal::Factory::NewFillerObject+816
71
+ 11: 00007FF604A31565 v8::internal::DateCache::Weekday+1349
72
+ 12: 00007FF604F4D961 v8::internal::SetupIsolateDelegate::SetupHeap+558193
73
+ 13: 00007FF5B03D1532
70
74
 
71
75
  */
72
76