rusteerio 1.1.1 → 1.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Cargo.lock +1 -1
- package/Cargo.toml +1 -1
- package/README.md +37 -37
- package/THIRD_PARTY_NOTICES.md +1 -1
- package/dist/{chunk-FPRP6UIX.mjs → chunk-HN63P3U7.mjs} +625 -401
- package/dist/chunk-HN63P3U7.mjs.map +1 -0
- package/dist/index.cjs +624 -400
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.mts +3 -2
- package/dist/index.d.ts +3 -2
- package/dist/index.mjs +1 -1
- package/dist/slim.cjs +555 -407
- package/dist/slim.cjs.map +1 -1
- package/dist/slim.d.mts +2 -2
- package/dist/slim.d.ts +2 -2
- package/dist/slim.mjs +1 -1
- package/dist/{types-Cs9UP1Vv.d.mts → types-CJsc0PH-.d.mts} +12 -0
- package/dist/{types-Cs9UP1Vv.d.ts → types-CJsc0PH-.d.ts} +12 -0
- package/dist/utils.d.mts +1 -1
- package/dist/utils.d.ts +1 -1
- package/native/src/arena.rs +163 -7
- package/native/src/attributes.rs +18 -1
- package/native/src/codec.rs +79 -0
- package/native/src/lib.rs +119 -30
- package/native/src/runtime.rs +23 -0
- package/native/src/snapshot.rs +117 -80
- package/native/src/tokenizer.rs +34 -20
- package/package.json +1 -1
- package/rusteerio.darwin-arm64.node +0 -0
- package/rusteerio.darwin-x64.node +0 -0
- package/rusteerio.linux-arm64-gnu.node +0 -0
- package/rusteerio.linux-arm64-musl.node +0 -0
- package/rusteerio.linux-x64-gnu.node +0 -0
- package/rusteerio.linux-x64-musl.node +0 -0
- package/rusteerio.win32-arm64-msvc.node +0 -0
- package/rusteerio.win32-x64-msvc.node +0 -0
- package/src/extract.ts +117 -24
- package/src/index.ts +27 -6
- package/src/match.ts +4 -4
- package/src/native.ts +4 -0
- package/src/nodes.ts +70 -10
- package/src/router.ts +80 -4
- package/src/selection.ts +18 -17
- package/src/snapshot.ts +37 -0
- package/src/transport.ts +1 -1
- package/dist/chunk-FPRP6UIX.mjs.map +0 -1
package/native/src/lib.rs
CHANGED
|
@@ -14,6 +14,8 @@ use napi_derive::napi;
|
|
|
14
14
|
use scraper::{Html, Node, Selector};
|
|
15
15
|
use serde_json::{Map, Value};
|
|
16
16
|
|
|
17
|
+
use codec::JsText;
|
|
18
|
+
|
|
17
19
|
#[cfg(target_os = "linux")]
|
|
18
20
|
#[global_allocator]
|
|
19
21
|
static ALLOCATOR: mimalloc::MiMalloc = mimalloc::MiMalloc;
|
|
@@ -158,6 +160,11 @@ pub struct NativeDocument {
|
|
|
158
160
|
// unreachable documents under native memory pressure.
|
|
159
161
|
content_bytes: Cell<usize>,
|
|
160
162
|
reported_heap_bytes: Cell<i64>,
|
|
163
|
+
/// Results of extraction plans run on the parsing thread, in request
|
|
164
|
+
/// order, until JavaScript takes them.
|
|
165
|
+
prepared_plans: RefCell<Option<Vec<Option<String>>>>,
|
|
166
|
+
/// A snapshot built on the parsing thread, until JavaScript takes it.
|
|
167
|
+
prepared_snapshot: RefCell<Option<snapshot::NodeSnapshot>>,
|
|
161
168
|
}
|
|
162
169
|
|
|
163
170
|
/// Node-API calls the finalizers of a module that declares the experimental
|
|
@@ -259,6 +266,7 @@ impl NativeDocument {
|
|
|
259
266
|
if arena.nodes.len() > POOL_RELEASE_NODES {
|
|
260
267
|
runtime::start_release_thread();
|
|
261
268
|
}
|
|
269
|
+
runtime::return_parse_pages(arena.nodes.len());
|
|
262
270
|
Self {
|
|
263
271
|
html: RefCell::new(None),
|
|
264
272
|
ids: RefCell::new(Vec::new()),
|
|
@@ -279,6 +287,8 @@ impl NativeDocument {
|
|
|
279
287
|
locations: RefCell::new(HashMap::new()),
|
|
280
288
|
content_bytes: Cell::new(content_bytes),
|
|
281
289
|
reported_heap_bytes: Cell::new(0),
|
|
290
|
+
prepared_plans: RefCell::new(None),
|
|
291
|
+
prepared_snapshot: RefCell::new(None),
|
|
282
292
|
}
|
|
283
293
|
}
|
|
284
294
|
pub(crate) fn set_diagnostic_records(&self, records: Vec<Value>) {
|
|
@@ -507,6 +517,7 @@ impl NativeDocument {
|
|
|
507
517
|
if ids.len() > POOL_RELEASE_NODES {
|
|
508
518
|
runtime::start_release_thread();
|
|
509
519
|
}
|
|
520
|
+
runtime::return_parse_pages(ids.len());
|
|
510
521
|
Self {
|
|
511
522
|
html: RefCell::new(Some(html)),
|
|
512
523
|
ids: RefCell::new(ids),
|
|
@@ -527,6 +538,8 @@ impl NativeDocument {
|
|
|
527
538
|
locations: RefCell::new(locations),
|
|
528
539
|
content_bytes: Cell::new(content_bytes),
|
|
529
540
|
reported_heap_bytes: Cell::new(0),
|
|
541
|
+
prepared_plans: RefCell::new(None),
|
|
542
|
+
prepared_snapshot: RefCell::new(None),
|
|
530
543
|
}
|
|
531
544
|
}
|
|
532
545
|
|
|
@@ -540,6 +553,14 @@ impl NativeDocument {
|
|
|
540
553
|
arena
|
|
541
554
|
}
|
|
542
555
|
|
|
556
|
+
/// The arena borrowed for a write that changes only attributes or data,
|
|
557
|
+
/// which keeps the name index valid.
|
|
558
|
+
pub(crate) fn arena_for_attribute_write(
|
|
559
|
+
&self,
|
|
560
|
+
) -> std::cell::RefMut<'_, Option<arena::DomArena>> {
|
|
561
|
+
self.arena.borrow_mut()
|
|
562
|
+
}
|
|
563
|
+
|
|
543
564
|
pub(crate) fn ensure_arena(&self) -> Result<()> {
|
|
544
565
|
let missing = self.arena.borrow().is_none();
|
|
545
566
|
if missing {
|
|
@@ -735,7 +756,9 @@ impl NativeDocument {
|
|
|
735
756
|
}
|
|
736
757
|
.into();
|
|
737
758
|
record.name = Some(element.name().into());
|
|
738
|
-
|
|
759
|
+
if !element.attrs.is_empty() {
|
|
760
|
+
*record.attribs = Map::with_capacity(element.attrs.len());
|
|
761
|
+
}
|
|
739
762
|
for (name, value) in &element.attrs {
|
|
740
763
|
record
|
|
741
764
|
.attribs
|
|
@@ -775,7 +798,7 @@ impl NativeDocument {
|
|
|
775
798
|
.map(|id| self.attribute_node(*id))
|
|
776
799
|
.collect::<Option<_>>()?;
|
|
777
800
|
let before: Vec<Map<String, Value>> =
|
|
778
|
-
nodes.iter().map(|node| node.attribs.clone()).collect();
|
|
801
|
+
nodes.iter().map(|node| (*node.attribs).clone()).collect();
|
|
779
802
|
let mut scratch = arena::DomArena::from_records(nodes, options.clone());
|
|
780
803
|
if self.options.get("__utf16Codec") == Some(&Value::Bool(true)) {
|
|
781
804
|
scratch.options["__utf16Codec"] = Value::Bool(true);
|
|
@@ -795,7 +818,7 @@ impl NativeDocument {
|
|
|
795
818
|
let node_ids = self.ids.borrow();
|
|
796
819
|
for (index, id) in targets.iter().enumerate() {
|
|
797
820
|
let after = &scratch.nodes[index].attribs;
|
|
798
|
-
if
|
|
821
|
+
if **after == before[index] {
|
|
799
822
|
continue;
|
|
800
823
|
}
|
|
801
824
|
let mut node = html.tree.get_mut(*node_ids.get(*id as usize)?)?;
|
|
@@ -1037,7 +1060,7 @@ impl NativeDocument {
|
|
|
1037
1060
|
}
|
|
1038
1061
|
}
|
|
1039
1062
|
self.ensure_arena()?;
|
|
1040
|
-
if let Some(arena) = self.
|
|
1063
|
+
if let Some(arena) = self.arena_for_attribute_write().as_mut() {
|
|
1041
1064
|
arena.set_attr(&ids, &name, value.as_deref());
|
|
1042
1065
|
}
|
|
1043
1066
|
Ok(())
|
|
@@ -1293,14 +1316,46 @@ impl NativeDocument {
|
|
|
1293
1316
|
.collect()
|
|
1294
1317
|
}
|
|
1295
1318
|
|
|
1296
|
-
|
|
1319
|
+
/// The results of the extraction plans `parseManyAsync` ran on the
|
|
1320
|
+
/// document's root while parsing it, each `null` if it failed; `null`
|
|
1321
|
+
/// after the first call.
|
|
1322
|
+
#[napi(
|
|
1323
|
+
js_name = "takePreparedPlans",
|
|
1324
|
+
ts_return_type = "Array<string | null> | null"
|
|
1325
|
+
)]
|
|
1326
|
+
pub fn take_prepared_plans(&self) -> Option<Vec<Option<JsText>>> {
|
|
1327
|
+
self.prepared_plans.borrow_mut().take().map(|results| {
|
|
1328
|
+
results
|
|
1329
|
+
.into_iter()
|
|
1330
|
+
.map(|result| result.map(JsText))
|
|
1331
|
+
.collect()
|
|
1332
|
+
})
|
|
1333
|
+
}
|
|
1334
|
+
|
|
1335
|
+
/// The snapshot `parseManyAsync` built while parsing the document, if it
|
|
1336
|
+
/// did; `null` after the first call, which discards it unless `is_wanted`.
|
|
1337
|
+
/// It describes the parsed tree, so JavaScript discards it on its first write.
|
|
1338
|
+
#[napi(js_name = "takePreparedSnapshot")]
|
|
1339
|
+
pub fn take_prepared_snapshot(&self, env: Env, is_wanted: bool) -> Result<Option<Object>> {
|
|
1340
|
+
let prepared = self.prepared_snapshot.borrow_mut().take();
|
|
1341
|
+
match prepared {
|
|
1342
|
+
Some(snapshot) if is_wanted => snapshot.into_js(&env).map(Some),
|
|
1343
|
+
_ => Ok(None),
|
|
1344
|
+
}
|
|
1345
|
+
}
|
|
1346
|
+
|
|
1347
|
+
#[napi(js_name = "text", ts_return_type = "string")]
|
|
1348
|
+
pub fn text_for_js(&self, ids: Vec<u32>) -> JsText {
|
|
1349
|
+
JsText(self.text(ids))
|
|
1350
|
+
}
|
|
1351
|
+
|
|
1297
1352
|
pub fn text(&self, ids: Vec<u32>) -> String {
|
|
1298
1353
|
self.text_ids(&ids)
|
|
1299
1354
|
}
|
|
1300
1355
|
|
|
1301
|
-
#[napi(js_name = "innerHtml")]
|
|
1302
|
-
pub fn inner_html_and_report(&self, env: Env, id: u32) ->
|
|
1303
|
-
self.reported(env, self.inner_html(id))
|
|
1356
|
+
#[napi(js_name = "innerHtml", ts_return_type = "string")]
|
|
1357
|
+
pub fn inner_html_and_report(&self, env: Env, id: u32) -> JsText {
|
|
1358
|
+
JsText(self.reported(env, self.inner_html(id)))
|
|
1304
1359
|
}
|
|
1305
1360
|
|
|
1306
1361
|
pub fn inner_html(&self, id: u32) -> String {
|
|
@@ -1312,20 +1367,24 @@ impl NativeDocument {
|
|
|
1312
1367
|
self.serialize_nodes(&[id], true)
|
|
1313
1368
|
}
|
|
1314
1369
|
|
|
1315
|
-
#[napi]
|
|
1370
|
+
#[napi(js_name = "outerHtml", ts_return_type = "string")]
|
|
1371
|
+
pub fn outer_html_for_js(&self, ids: Vec<u32>) -> JsText {
|
|
1372
|
+
JsText(self.outer_html(ids))
|
|
1373
|
+
}
|
|
1374
|
+
|
|
1316
1375
|
pub fn outer_html(&self, ids: Vec<u32>) -> String {
|
|
1317
1376
|
self.serialize_nodes(&ids, false)
|
|
1318
1377
|
}
|
|
1319
1378
|
|
|
1320
1379
|
/// Serializes nodes as the `render` command does, returning the markup
|
|
1321
1380
|
/// itself rather than markup encoded as JSON.
|
|
1322
|
-
#[napi(js_name = "render")]
|
|
1381
|
+
#[napi(js_name = "render", ts_return_type = "string")]
|
|
1323
1382
|
pub fn render_and_report(
|
|
1324
1383
|
&self,
|
|
1325
1384
|
env: Env,
|
|
1326
1385
|
ids: Vec<u32>,
|
|
1327
1386
|
options_json: Option<String>,
|
|
1328
|
-
) -> Result<
|
|
1387
|
+
) -> Result<JsText> {
|
|
1329
1388
|
self.maybe_collect();
|
|
1330
1389
|
let overrides = match options_json {
|
|
1331
1390
|
Some(json) => Some(serde_json::from_str::<Value>(&json).map_err(json_error)?),
|
|
@@ -1333,7 +1392,7 @@ impl NativeDocument {
|
|
|
1333
1392
|
}
|
|
1334
1393
|
.filter(Value::is_object);
|
|
1335
1394
|
let rendered = self.render_nodes(&ids, overrides)?;
|
|
1336
|
-
Ok(self.reported(env, rendered))
|
|
1395
|
+
Ok(JsText(self.reported(env, rendered)))
|
|
1337
1396
|
}
|
|
1338
1397
|
|
|
1339
1398
|
fn render_nodes(&self, ids: &[u32], overrides: Option<Value>) -> Result<String> {
|
|
@@ -1364,7 +1423,11 @@ impl NativeDocument {
|
|
|
1364
1423
|
Ok(arena.serialize(ids, &arena::normalized_options(options)))
|
|
1365
1424
|
}
|
|
1366
1425
|
|
|
1367
|
-
#[napi]
|
|
1426
|
+
#[napi(js_name = "attr", ts_return_type = "string | null")]
|
|
1427
|
+
pub fn attr_for_js(&self, id: u32, name: String) -> Option<JsText> {
|
|
1428
|
+
self.attr(id, name).map(JsText)
|
|
1429
|
+
}
|
|
1430
|
+
|
|
1368
1431
|
pub fn attr(&self, id: u32, name: String) -> Option<String> {
|
|
1369
1432
|
self.attribute(id, &name)
|
|
1370
1433
|
}
|
|
@@ -1382,9 +1445,9 @@ impl NativeDocument {
|
|
|
1382
1445
|
.flatten()
|
|
1383
1446
|
}
|
|
1384
1447
|
|
|
1385
|
-
#[napi(js_name = "exportTree")]
|
|
1386
|
-
pub fn export_tree_and_report(&self, env: Env) -> Result<
|
|
1387
|
-
self.reported(env, self.export_tree())
|
|
1448
|
+
#[napi(js_name = "exportTree", ts_return_type = "string")]
|
|
1449
|
+
pub fn export_tree_and_report(&self, env: Env) -> Result<JsText> {
|
|
1450
|
+
self.reported(env, self.export_tree()).map(JsText)
|
|
1388
1451
|
}
|
|
1389
1452
|
|
|
1390
1453
|
pub fn export_tree(&self) -> Result<String> {
|
|
@@ -1412,9 +1475,9 @@ impl NativeDocument {
|
|
|
1412
1475
|
built.into_js(&env)
|
|
1413
1476
|
}
|
|
1414
1477
|
|
|
1415
|
-
#[napi(js_name = "node")]
|
|
1416
|
-
pub fn node_and_report(&self, env: Env, id: u32) -> Result<
|
|
1417
|
-
self.reported(env, self.node(id))
|
|
1478
|
+
#[napi(js_name = "node", ts_return_type = "string")]
|
|
1479
|
+
pub fn node_and_report(&self, env: Env, id: u32) -> Result<JsText> {
|
|
1480
|
+
self.reported(env, self.node(id)).map(JsText)
|
|
1418
1481
|
}
|
|
1419
1482
|
|
|
1420
1483
|
pub fn node(&self, id: u32) -> Result<String> {
|
|
@@ -1563,9 +1626,9 @@ impl NativeDocument {
|
|
|
1563
1626
|
.map_err(json_error)
|
|
1564
1627
|
}
|
|
1565
1628
|
|
|
1566
|
-
#[napi(js_name = "op")]
|
|
1567
|
-
pub fn op_and_report(&self, env: Env, command: String, args_json: String) -> Result<
|
|
1568
|
-
self.reported(env, self.op(command, args_json))
|
|
1629
|
+
#[napi(js_name = "op", ts_return_type = "string")]
|
|
1630
|
+
pub fn op_and_report(&self, env: Env, command: String, args_json: String) -> Result<JsText> {
|
|
1631
|
+
self.reported(env, self.op(command, args_json)).map(JsText)
|
|
1569
1632
|
}
|
|
1570
1633
|
|
|
1571
1634
|
pub fn op(&self, command: String, args_json: String) -> Result<String> {
|
|
@@ -1671,7 +1734,11 @@ impl NativeDocument {
|
|
|
1671
1734
|
}
|
|
1672
1735
|
}
|
|
1673
1736
|
self.ensure_arena()?;
|
|
1674
|
-
let mut arena =
|
|
1737
|
+
let mut arena = if attributes::keeps_element_names(&command) {
|
|
1738
|
+
self.arena_for_attribute_write()
|
|
1739
|
+
} else {
|
|
1740
|
+
self.arena_for_write()
|
|
1741
|
+
};
|
|
1675
1742
|
let arena = arena.as_mut().unwrap();
|
|
1676
1743
|
let previous_options = arena.options.clone();
|
|
1677
1744
|
if let Some(options) = args.get("options").filter(|options| options.is_object()) {
|
|
@@ -2058,7 +2125,9 @@ fn convert_node(
|
|
|
2058
2125
|
.insert("prefix".into(), Value::String(String::from(&**prefix)));
|
|
2059
2126
|
}
|
|
2060
2127
|
let (mut namespaces, mut prefixes) = (Map::new(), Map::new());
|
|
2061
|
-
|
|
2128
|
+
if !element.attrs.is_empty() {
|
|
2129
|
+
*record.attribs = Map::with_capacity(element.attrs.len());
|
|
2130
|
+
}
|
|
2062
2131
|
for (name, value) in &element.attrs {
|
|
2063
2132
|
let key = String::from(&*name.local);
|
|
2064
2133
|
if !name.ns.is_empty() {
|
|
@@ -2299,6 +2368,9 @@ pub struct ParseManyTask {
|
|
|
2299
2368
|
is_document: bool,
|
|
2300
2369
|
threads: usize,
|
|
2301
2370
|
options: Value,
|
|
2371
|
+
/// Extraction plans to run on each root after parsing; with them, each
|
|
2372
|
+
/// document also gets its snapshot built on the parsing thread.
|
|
2373
|
+
plans: Option<Vec<Value>>,
|
|
2302
2374
|
}
|
|
2303
2375
|
|
|
2304
2376
|
#[napi]
|
|
@@ -2308,12 +2380,24 @@ impl Task for ParseManyTask {
|
|
|
2308
2380
|
|
|
2309
2381
|
fn compute(&mut self) -> Result<Self::Output> {
|
|
2310
2382
|
runtime::parallel_map(&self.htmls, self.threads, |html| {
|
|
2311
|
-
|
|
2312
|
-
html,
|
|
2313
|
-
|
|
2314
|
-
|
|
2315
|
-
|
|
2316
|
-
|
|
2383
|
+
let document =
|
|
2384
|
+
NativeDocument::parse_options(html, self.is_document, self.options.clone(), None);
|
|
2385
|
+
if let Some(plans) = &self.plans {
|
|
2386
|
+
let results = plans
|
|
2387
|
+
.iter()
|
|
2388
|
+
.map(|plan| {
|
|
2389
|
+
let args =
|
|
2390
|
+
serde_json::json!({"ids": [0], "plan": plan, "options": self.options});
|
|
2391
|
+
document.op("extractPlan".into(), args.to_string()).ok()
|
|
2392
|
+
})
|
|
2393
|
+
.collect();
|
|
2394
|
+
*document.prepared_plans.borrow_mut() = Some(results);
|
|
2395
|
+
// With source locations the snapshot is built from the mutable tree instead.
|
|
2396
|
+
if document.source.borrow().is_none() {
|
|
2397
|
+
*document.prepared_snapshot.borrow_mut() = Some(snapshot::build(&document));
|
|
2398
|
+
}
|
|
2399
|
+
}
|
|
2400
|
+
Ok(document)
|
|
2317
2401
|
})
|
|
2318
2402
|
}
|
|
2319
2403
|
|
|
@@ -2328,8 +2412,12 @@ pub fn parse_many_async(
|
|
|
2328
2412
|
is_document: bool,
|
|
2329
2413
|
threads: Option<u32>,
|
|
2330
2414
|
options_json: Option<String>,
|
|
2415
|
+
plans_json: Option<String>,
|
|
2331
2416
|
) -> Result<AsyncTask<ParseManyTask>> {
|
|
2332
2417
|
let threads = thread_count(threads, htmls.len())?;
|
|
2418
|
+
let plans = plans_json
|
|
2419
|
+
.map(|json| serde_json::from_str(&json).map_err(json_error))
|
|
2420
|
+
.transpose()?;
|
|
2333
2421
|
Ok(AsyncTask::new(ParseManyTask {
|
|
2334
2422
|
htmls,
|
|
2335
2423
|
is_document,
|
|
@@ -2337,6 +2425,7 @@ pub fn parse_many_async(
|
|
|
2337
2425
|
options: options_json
|
|
2338
2426
|
.and_then(|value| serde_json::from_str(&value).ok())
|
|
2339
2427
|
.unwrap_or_else(|| serde_json::json!({})),
|
|
2428
|
+
plans,
|
|
2340
2429
|
}))
|
|
2341
2430
|
}
|
|
2342
2431
|
|
package/native/src/runtime.rs
CHANGED
|
@@ -183,6 +183,29 @@ static RELEASE_THREAD: OnceLock<mpsc::Sender<Box<dyn Send>>> = OnceLock::new();
|
|
|
183
183
|
/// exists, so freeing large trees does not stall JavaScript. Documents large
|
|
184
184
|
/// enough to be released later start it when they are created, because
|
|
185
185
|
/// finalizers must not create threads.
|
|
186
|
+
/// Trees with at least this many nodes return the parse's freed memory to
|
|
187
|
+
/// the system once built.
|
|
188
|
+
#[cfg(target_os = "linux")]
|
|
189
|
+
const RETURN_PAGES_NODES: usize = 100_000;
|
|
190
|
+
|
|
191
|
+
#[cfg(target_os = "linux")]
|
|
192
|
+
extern "C" {
|
|
193
|
+
fn mi_collect(force: bool);
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
/// Building a large tree grows and then frees big buffers, such as a node
|
|
197
|
+
/// vector that doubled several times. mimalloc keeps freed pages resident for
|
|
198
|
+
/// reuse; after a large parse, return them on the parsing thread instead.
|
|
199
|
+
pub(crate) fn return_parse_pages(nodes: usize) {
|
|
200
|
+
#[cfg(target_os = "linux")]
|
|
201
|
+
if nodes >= RETURN_PAGES_NODES {
|
|
202
|
+
// SAFETY: mimalloc's own collection frees only memory no block uses.
|
|
203
|
+
unsafe { mi_collect(true) };
|
|
204
|
+
}
|
|
205
|
+
#[cfg(not(target_os = "linux"))]
|
|
206
|
+
let _ = nodes;
|
|
207
|
+
}
|
|
208
|
+
|
|
186
209
|
pub(crate) fn start_release_thread() {
|
|
187
210
|
RELEASE_THREAD.get_or_init(|| {
|
|
188
211
|
let (sender, receiver) = mpsc::channel::<Box<dyn Send>>();
|
package/native/src/snapshot.rs
CHANGED
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
//! which frees it as soon as the snapshot is collected; an external buffer
|
|
16
16
|
//! would add finalizer bookkeeping and wait for a later event-loop turn.
|
|
17
17
|
use html5ever::{LocalName, Namespace};
|
|
18
|
-
use napi::{Env, JsObject, JsTypedArray,
|
|
18
|
+
use napi::{check_status, sys, Env, JsObject, JsTypedArray, NapiValue, Result};
|
|
19
19
|
use scraper::Node;
|
|
20
20
|
use serde_json::{Map, Value};
|
|
21
21
|
|
|
@@ -36,6 +36,17 @@ const EMPTY_ATTRIBUTE_MAPS: u32 = 8;
|
|
|
36
36
|
/// The record cannot be represented here; JavaScript reads it individually.
|
|
37
37
|
const OPAQUE: u32 = 16;
|
|
38
38
|
const RELEASED: u32 = 32;
|
|
39
|
+
const KINDS: [&str; 8] = [
|
|
40
|
+
"tag",
|
|
41
|
+
"text",
|
|
42
|
+
"comment",
|
|
43
|
+
"root",
|
|
44
|
+
"script",
|
|
45
|
+
"style",
|
|
46
|
+
"directive",
|
|
47
|
+
"cdata",
|
|
48
|
+
];
|
|
49
|
+
const SHORT_NAME_SLOTS: usize = 256;
|
|
39
50
|
|
|
40
51
|
pub(crate) struct NodeSnapshot {
|
|
41
52
|
is_encoded: bool,
|
|
@@ -58,90 +69,74 @@ impl NodeSnapshot {
|
|
|
58
69
|
u32::from(self.is_encoded),
|
|
59
70
|
];
|
|
60
71
|
let length = header.len() + parts.iter().map(|part| part.len()).sum::<usize>();
|
|
61
|
-
let
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
for part in std::iter::once(&header[..]).chain(parts.iter().map(|part| part.as_slice())) {
|
|
70
|
-
words[offset..offset + part.len()].copy_from_slice(part);
|
|
71
|
-
offset += part.len();
|
|
72
|
-
}
|
|
73
|
-
let data: JsTypedArray =
|
|
74
|
-
buffer
|
|
75
|
-
.into_raw()
|
|
76
|
-
.into_typedarray(TypedArrayType::Uint32, length, 0)?;
|
|
72
|
+
let data = uninitialized_words(env, length, |words| {
|
|
73
|
+
let mut offset = 0;
|
|
74
|
+
for part in std::iter::once(&header[..]).chain(parts.iter().map(|part| part.as_slice()))
|
|
75
|
+
{
|
|
76
|
+
words[offset..offset + part.len()].copy_from_slice(part);
|
|
77
|
+
offset += part.len();
|
|
78
|
+
}
|
|
79
|
+
})?;
|
|
77
80
|
let mut object = env.create_object()?;
|
|
78
81
|
object.set_named_property("data", data)?;
|
|
79
|
-
|
|
80
|
-
// character fits, which also keeps V8's compact one-byte string.
|
|
81
|
-
let text = if self.text.is_ascii() {
|
|
82
|
-
env.create_string_latin1(self.text.as_bytes())?
|
|
83
|
-
} else if let Some(latin1) = latin1_bytes(&self.text) {
|
|
84
|
-
env.create_string_latin1(&latin1)?
|
|
85
|
-
} else {
|
|
86
|
-
env.create_string_utf16(&utf16_units(&self.text))?
|
|
87
|
-
};
|
|
88
|
-
object.set_named_property("text", text)?;
|
|
82
|
+
object.set_named_property("text", crate::codec::js_string(env, &self.text)?)?;
|
|
89
83
|
Ok(object)
|
|
90
84
|
}
|
|
91
85
|
}
|
|
92
86
|
|
|
93
|
-
/// `
|
|
94
|
-
///
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
let
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
),
|
|
134
|
-
};
|
|
135
|
-
if scalar < 0x1_0000 {
|
|
136
|
-
units.push(scalar as u16);
|
|
137
|
-
} else {
|
|
138
|
-
let offset = scalar - 0x1_0000;
|
|
139
|
-
units.push(0xD800 | (offset >> 10) as u16);
|
|
140
|
-
units.push(0xDC00 | (offset & 0x3FF) as u16);
|
|
87
|
+
/// A `Uint32Array` of `length` words that `fill` writes. A Node buffer is
|
|
88
|
+
/// used for its storage because, unlike an `ArrayBuffer`, it is not
|
|
89
|
+
/// zero-filled first; `fill` writes every word before JavaScript can read it.
|
|
90
|
+
fn uninitialized_words(
|
|
91
|
+
env: &Env,
|
|
92
|
+
length: usize,
|
|
93
|
+
fill: impl FnOnce(&mut [u32]),
|
|
94
|
+
) -> Result<JsTypedArray> {
|
|
95
|
+
let raw_env = env.raw();
|
|
96
|
+
let mut buffer = std::ptr::null_mut();
|
|
97
|
+
let mut memory = std::ptr::null_mut();
|
|
98
|
+
// SAFETY: Node-API calls on the current environment with valid out
|
|
99
|
+
// pointers. `memory` is the buffer's own allocation of `length` words,
|
|
100
|
+
// checked to be aligned, and the buffer stays alive in this handle scope.
|
|
101
|
+
unsafe {
|
|
102
|
+
check_status!(sys::napi_create_buffer(
|
|
103
|
+
raw_env,
|
|
104
|
+
length * 4,
|
|
105
|
+
&mut memory,
|
|
106
|
+
&mut buffer
|
|
107
|
+
))?;
|
|
108
|
+
let mut kind = 0;
|
|
109
|
+
let mut bytes = 0;
|
|
110
|
+
let mut view_memory = std::ptr::null_mut();
|
|
111
|
+
let mut arraybuffer = std::ptr::null_mut();
|
|
112
|
+
let mut byte_offset = 0;
|
|
113
|
+
check_status!(sys::napi_get_typedarray_info(
|
|
114
|
+
raw_env,
|
|
115
|
+
buffer,
|
|
116
|
+
&mut kind,
|
|
117
|
+
&mut bytes,
|
|
118
|
+
&mut view_memory,
|
|
119
|
+
&mut arraybuffer,
|
|
120
|
+
&mut byte_offset,
|
|
121
|
+
))?;
|
|
122
|
+
if !byte_offset.is_multiple_of(4)
|
|
123
|
+
|| !(memory as usize).is_multiple_of(4)
|
|
124
|
+
|| bytes < length * 4
|
|
125
|
+
{
|
|
126
|
+
return Err(napi::Error::from_reason("Unaligned snapshot buffer"));
|
|
141
127
|
}
|
|
142
|
-
|
|
128
|
+
let mut words = std::ptr::null_mut();
|
|
129
|
+
check_status!(sys::napi_create_typedarray(
|
|
130
|
+
raw_env,
|
|
131
|
+
sys::TypedarrayType::uint32_array,
|
|
132
|
+
length,
|
|
133
|
+
arraybuffer,
|
|
134
|
+
byte_offset,
|
|
135
|
+
&mut words,
|
|
136
|
+
))?;
|
|
137
|
+
fill(std::slice::from_raw_parts_mut(memory.cast::<u32>(), length));
|
|
138
|
+
Ok(JsTypedArray::from_raw_unchecked(raw_env, words))
|
|
143
139
|
}
|
|
144
|
-
units
|
|
145
140
|
}
|
|
146
141
|
|
|
147
142
|
/// The UTF-16 length of `text`: one unit per character, two for characters
|
|
@@ -160,6 +155,11 @@ struct Builder {
|
|
|
160
155
|
text: String,
|
|
161
156
|
units: u32,
|
|
162
157
|
interned: IdMap<String, u32>,
|
|
158
|
+
/// References of the common node types, resolved without hashing.
|
|
159
|
+
kinds: [Option<u32>; KINDS.len()],
|
|
160
|
+
/// Recently interned names of up to eight bytes, keyed by their bytes:
|
|
161
|
+
/// element and attribute names repeat across nodes.
|
|
162
|
+
short_names: Box<[(u64, u8, u32); SHORT_NAME_SLOTS]>,
|
|
163
163
|
/// html5ever names are atoms that hash and compare by identity.
|
|
164
164
|
local_names: IdMap<LocalName, u32>,
|
|
165
165
|
namespaces: IdMap<Namespace, u32>,
|
|
@@ -179,6 +179,8 @@ pub(crate) fn build(document: &NativeDocument) -> NodeSnapshot {
|
|
|
179
179
|
text: String::with_capacity(document.content_bytes.get()),
|
|
180
180
|
units: 0,
|
|
181
181
|
interned: IdMap::with_capacity_and_hasher(32, Default::default()),
|
|
182
|
+
kinds: [None; KINDS.len()],
|
|
183
|
+
short_names: Box::new([(0, 0, 0); SHORT_NAME_SLOTS]),
|
|
182
184
|
local_names: IdMap::with_capacity_and_hasher(32, Default::default()),
|
|
183
185
|
namespaces: IdMap::default(),
|
|
184
186
|
codec,
|
|
@@ -245,6 +247,40 @@ impl Builder {
|
|
|
245
247
|
(self.strings.len() / 2 - 1) as u32
|
|
246
248
|
}
|
|
247
249
|
|
|
250
|
+
fn kind(&mut self, kind: &str) -> u32 {
|
|
251
|
+
let Some(slot) = KINDS.iter().position(|known| *known == kind) else {
|
|
252
|
+
return self.interned(kind);
|
|
253
|
+
};
|
|
254
|
+
if let Some(reference) = self.kinds[slot] {
|
|
255
|
+
return reference;
|
|
256
|
+
}
|
|
257
|
+
let reference = self.interned(kind);
|
|
258
|
+
self.kinds[slot] = Some(reference);
|
|
259
|
+
reference
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
fn interned_name(&mut self, name: &str) -> u32 {
|
|
263
|
+
let bytes = name.as_bytes();
|
|
264
|
+
if bytes.len() > 8 {
|
|
265
|
+
return self.interned(name);
|
|
266
|
+
}
|
|
267
|
+
let mut word = [0; 8];
|
|
268
|
+
word[..bytes.len()].copy_from_slice(bytes);
|
|
269
|
+
let key = u64::from_le_bytes(word);
|
|
270
|
+
let length = bytes.len() as u8;
|
|
271
|
+
let slot = (key
|
|
272
|
+
.wrapping_add(u64::from(length))
|
|
273
|
+
.wrapping_mul(0x9E37_79B9_7F4A_7C15)
|
|
274
|
+
>> 56) as usize;
|
|
275
|
+
let (cached_key, cached_length, cached) = self.short_names[slot];
|
|
276
|
+
if cached != 0 && cached_key == key && cached_length == length {
|
|
277
|
+
return cached - 1;
|
|
278
|
+
}
|
|
279
|
+
let reference = self.interned(name);
|
|
280
|
+
self.short_names[slot] = (key, length, reference + 1);
|
|
281
|
+
reference
|
|
282
|
+
}
|
|
283
|
+
|
|
248
284
|
fn interned(&mut self, value: &str) -> u32 {
|
|
249
285
|
if let Some(reference) = self.interned.get(value) {
|
|
250
286
|
return *reference;
|
|
@@ -280,7 +316,7 @@ impl Builder {
|
|
|
280
316
|
|
|
281
317
|
fn encode(&mut self, node: &DomNode) {
|
|
282
318
|
let mut fields = [0; STRIDE];
|
|
283
|
-
fields[0] = self.
|
|
319
|
+
fields[0] = self.kind(&node.kind);
|
|
284
320
|
fields[1] = pointer(node.parent);
|
|
285
321
|
fields[2] = pointer(node.prev);
|
|
286
322
|
fields[3] = pointer(node.next);
|
|
@@ -290,7 +326,7 @@ impl Builder {
|
|
|
290
326
|
fields[6] = node
|
|
291
327
|
.name
|
|
292
328
|
.as_deref()
|
|
293
|
-
.map_or(0, |name| self.
|
|
329
|
+
.map_or(0, |name| self.interned_name(name) + 1);
|
|
294
330
|
let mut flags = 0;
|
|
295
331
|
match node.data.as_deref() {
|
|
296
332
|
Some(data) if data.len() > LARGE_TEXT_BYTES => flags |= OPAQUE,
|
|
@@ -313,7 +349,7 @@ impl Builder {
|
|
|
313
349
|
flags |= OPAQUE;
|
|
314
350
|
break;
|
|
315
351
|
};
|
|
316
|
-
let name = self.
|
|
352
|
+
let name = self.interned_name(name);
|
|
317
353
|
let value = self.string(value);
|
|
318
354
|
self.attributes.extend_from_slice(&[name, value]);
|
|
319
355
|
}
|
|
@@ -492,6 +528,7 @@ impl Builder {
|
|
|
492
528
|
#[cfg(test)]
|
|
493
529
|
mod tests {
|
|
494
530
|
use super::*;
|
|
531
|
+
use crate::codec::{latin1_bytes, utf16_units};
|
|
495
532
|
|
|
496
533
|
#[test]
|
|
497
534
|
fn transcodes_text_like_the_standard_library() {
|