xml-tokenizer 0.0.44 → 0.0.46
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +170 -132
- package/dist/esm/tokenizer/utils.js +1 -1
- package/package.json +24 -11
package/README.md
CHANGED
|
@@ -17,187 +17,225 @@
|
|
|
17
17
|
</a>
|
|
18
18
|
</p>
|
|
19
19
|
|
|
20
|
-
|
|
20
|
+
`xml-tokenizer` is a callback-based XML, HTML, and SVG tokenizer for TypeScript. It emits typed tokens, supports early exit when you have the data you need, and includes small helpers for path selection and object conversion when a full DOM would be unnecessary.
|
|
21
21
|
|
|
22
|
-
|
|
23
|
-
|
|
22
|
+
- Emit `ElementStart`, `Attribute`, `Text`, `Cdata`, and other tokens without building a tree first
|
|
23
|
+
- Pick XML document mode, HTML, or SVG parsing behavior with `xmlConfig`, `htmlConfig`, and `svgConfig`
|
|
24
|
+
- Stop parsing from inside the callback with `stream.goToEnd()`
|
|
25
|
+
- Select matching token ranges with object-based path selectors
|
|
26
|
+
- Convert markup into nested or simplified objects when a tree shape is more convenient
|
|
24
27
|
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
28
|
+
```ts
|
|
29
|
+
import { htmlConfig, tokenize, type TXmlToken } from 'xml-tokenizer';
|
|
30
|
+
|
|
31
|
+
let title: string | null = null;
|
|
32
|
+
let insideTitle = false;
|
|
33
|
+
|
|
34
|
+
tokenize(
|
|
35
|
+
'<html><head><title>Hello</title></head></html>',
|
|
36
|
+
(token: TXmlToken, stream) => {
|
|
37
|
+
if (token.type === 'ElementStart' && token.local === 'title') {
|
|
38
|
+
insideTitle = true;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
if (insideTitle && token.type === 'Text') {
|
|
42
|
+
const text = token.text.trim();
|
|
43
|
+
if (text !== '') {
|
|
44
|
+
title = text;
|
|
45
|
+
stream.goToEnd();
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
},
|
|
49
|
+
htmlConfig
|
|
50
|
+
);
|
|
31
51
|
|
|
32
|
-
|
|
52
|
+
console.log(title); // Hello
|
|
53
|
+
```
|
|
33
54
|
|
|
34
|
-
|
|
55
|
+
## Install
|
|
35
56
|
|
|
36
|
-
|
|
57
|
+
```bash
|
|
58
|
+
npm install xml-tokenizer
|
|
59
|
+
```
|
|
37
60
|
|
|
38
|
-
|
|
61
|
+
## Usage
|
|
39
62
|
|
|
40
|
-
|
|
63
|
+
Use `tokenize` when you want to process markup as a stream:
|
|
41
64
|
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
65
|
+
```ts
|
|
66
|
+
import { tokenize, xmlConfig, type TXmlToken } from 'xml-tokenizer';
|
|
67
|
+
|
|
68
|
+
tokenize(
|
|
69
|
+
'<book id="1"><title>Dune</title></book>',
|
|
70
|
+
(token: TXmlToken) => {
|
|
71
|
+
switch (token.type) {
|
|
72
|
+
case 'ElementStart':
|
|
73
|
+
console.log('element', token.local);
|
|
74
|
+
break;
|
|
75
|
+
case 'Attribute':
|
|
76
|
+
console.log('attribute', token.local, token.value);
|
|
77
|
+
break;
|
|
78
|
+
case 'Text':
|
|
79
|
+
console.log('text', token.text.trim());
|
|
80
|
+
break;
|
|
81
|
+
}
|
|
82
|
+
},
|
|
83
|
+
xmlConfig
|
|
84
|
+
);
|
|
85
|
+
```
|
|
45
86
|
|
|
46
|
-
|
|
87
|
+
Use `select` when you only care about matching paths:
|
|
47
88
|
|
|
48
89
|
```ts
|
|
49
|
-
import { select,
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
// Or, parse XML to a stream of tokens
|
|
58
|
-
tokenize('<p>Hello World</p>', (token) => {
|
|
59
|
-
switch (token.type) {
|
|
60
|
-
case 'ElementStart':
|
|
61
|
-
console.log('Start of element:', token);
|
|
62
|
-
break;
|
|
63
|
-
case 'Text':
|
|
64
|
-
console.log('Text content:', token.text);
|
|
65
|
-
break;
|
|
66
|
-
// Handle other token types as needed
|
|
67
|
-
default:
|
|
68
|
-
console.log('Token:', token);
|
|
69
|
-
}
|
|
70
|
-
});
|
|
90
|
+
import { select, xmlConfig } from 'xml-tokenizer';
|
|
91
|
+
|
|
92
|
+
const xml = `
|
|
93
|
+
<bookstore>
|
|
94
|
+
<book category="COOKING"><title>Everyday Italian</title></book>
|
|
95
|
+
</bookstore>
|
|
96
|
+
`;
|
|
71
97
|
|
|
72
|
-
// Or, stream only a selection of tokens
|
|
73
98
|
select(
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
99
|
+
xml,
|
|
100
|
+
[
|
|
101
|
+
[
|
|
102
|
+
{ axis: 'child', local: 'bookstore' },
|
|
103
|
+
{ axis: 'child', local: 'book', attributes: [{ local: 'category', value: 'COOKING' }] }
|
|
104
|
+
]
|
|
105
|
+
],
|
|
106
|
+
(token, stream) => {
|
|
107
|
+
if (token.type === 'Text') {
|
|
108
|
+
console.log(token.text.trim());
|
|
109
|
+
stream.goToEnd();
|
|
110
|
+
}
|
|
111
|
+
},
|
|
112
|
+
xmlConfig
|
|
84
113
|
);
|
|
85
114
|
```
|
|
86
115
|
|
|
87
|
-
|
|
116
|
+
Use the object helpers when you want a small tree representation:
|
|
88
117
|
|
|
89
|
-
|
|
118
|
+
```ts
|
|
119
|
+
import { htmlToMarkdown, xmlToObject, xmlToSimplifiedObject } from 'xml-tokenizer';
|
|
90
120
|
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
- **ElementStart**: `<ns:elem`
|
|
95
|
-
- **Attribute**: `ns:attr="value"`
|
|
96
|
-
- **ElementEnd**:
|
|
97
|
-
- Open: `>`
|
|
98
|
-
- Close: `</ns:name>`
|
|
99
|
-
- Empty: `/>`
|
|
100
|
-
- **Text**: Text content between elements, including whitespace.
|
|
101
|
-
- **Cdata**: `<![CDATA[text]]>`
|
|
121
|
+
const tree = xmlToObject('<book id="1"><title>Dune</title></book>');
|
|
122
|
+
const simplified = xmlToSimplifiedObject('<book id="1"><title>Dune</title></book>');
|
|
123
|
+
```
|
|
102
124
|
|
|
103
|
-
|
|
125
|
+
`tree` contains nested nodes:
|
|
104
126
|
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
127
|
+
```ts
|
|
128
|
+
const tree = {
|
|
129
|
+
local: 'book',
|
|
130
|
+
attributes: [{ local: 'id', value: '1' }],
|
|
131
|
+
content: [{ local: 'title', attributes: [], content: ['Dune'] }]
|
|
132
|
+
};
|
|
133
|
+
```
|
|
109
134
|
|
|
110
|
-
|
|
135
|
+
`simplified` stores element names under underscored keys:
|
|
111
136
|
|
|
112
|
-
|
|
137
|
+
```ts
|
|
138
|
+
const simplified = {
|
|
139
|
+
_book: [
|
|
140
|
+
{
|
|
141
|
+
attributes: { id: '1' },
|
|
142
|
+
_title: [{ text: 'Dune' }]
|
|
143
|
+
}
|
|
144
|
+
]
|
|
145
|
+
};
|
|
146
|
+
```
|
|
113
147
|
|
|
114
|
-
|
|
148
|
+
## Configs
|
|
115
149
|
|
|
116
|
-
|
|
117
|
-
| -------------------- | ------------------------------- | ------------- | ------------- | -------------- | ------------------------------ |
|
|
118
|
-
| xml-tokenizer | 46.87 | 19.47 | 24.57 | 21.33 | ±2.06% |
|
|
119
|
-
| xml-tokenizer (dist) | 53.70 | 17.31 | 25.20 | 18.62 | ±3.28% |
|
|
120
|
-
| xml-tokenizer (npm) | 163.00 | 5.03 | 8.50 | 6.13 | ±2.32% |
|
|
121
|
-
| fast-xml-parser | 66.00 | 14.01 | 20.73 | 15.15 | ±3.34% |
|
|
122
|
-
| txml | 234.52 | 3.38 | 7.61 | 4.26 | ±4.00% |
|
|
123
|
-
| xml2js | 36.21 | 25.58 | 37.28 | 27.61 | ±4.39% |
|
|
150
|
+
Choose the config that matches the input:
|
|
124
151
|
|
|
125
|
-
|
|
152
|
+
| Config | Use for |
|
|
153
|
+
| ------------ | ------------------------------------ |
|
|
154
|
+
| `xmlConfig` | XML document mode |
|
|
155
|
+
| `htmlConfig` | HTML with raw text and void elements |
|
|
156
|
+
| `svgConfig` | SVG fragments and documents |
|
|
126
157
|
|
|
127
|
-
|
|
128
|
-
| ------------------- | ------------------------------- | ------------- | ------------- | -------------- | ------------------------------ |
|
|
129
|
-
| xml-tokenizer | 53.03 | 18.30 | 19.45 | 18.86 | ±0.81% |
|
|
130
|
-
| xml-tokenizer (npm) | 166.61 | 5.62 | 7.16 | 6.00 | ±0.88% |
|
|
131
|
-
| saxen | 500.99 | 1.83 | 4.79 | 2.00 | ±1.52% |
|
|
132
|
-
| sax | 64.44 | 14.96 | 16.34 | 15.52 | ±0.67% |
|
|
158
|
+
All configs are `TXmlStreamOptions`, so you can pass custom options when you need different parser behavior:
|
|
133
159
|
|
|
134
|
-
|
|
160
|
+
```ts
|
|
161
|
+
tokenize(markup, onToken, {
|
|
162
|
+
...htmlConfig,
|
|
163
|
+
contextSliceSize: 80
|
|
164
|
+
});
|
|
165
|
+
```
|
|
135
166
|
|
|
136
|
-
|
|
167
|
+
## Tokens
|
|
137
168
|
|
|
138
|
-
|
|
139
|
-
pnpm run bench
|
|
140
|
-
```
|
|
169
|
+
`tokenize` can emit these token types:
|
|
141
170
|
|
|
142
|
-
|
|
171
|
+
| Token | Example |
|
|
172
|
+
| ----------------------- | ------------------------ |
|
|
173
|
+
| `ProcessingInstruction` | `<?target content?>` |
|
|
174
|
+
| `Comment` | `<!-- text -->` |
|
|
175
|
+
| `EntityDeclaration` | `<!ENTITY name "value">` |
|
|
176
|
+
| `ElementStart` | `<book` |
|
|
177
|
+
| `Attribute` | `id="1"` |
|
|
178
|
+
| `ElementEnd` | `>`, `</book>`, or `/>` |
|
|
179
|
+
| `Text` | Text between elements |
|
|
180
|
+
| `Cdata` | `<![CDATA[text]]>` |
|
|
143
181
|
|
|
144
|
-
|
|
182
|
+
Structure-sensitive code should handle `ElementStart`, `ElementEnd`, and `Attribute` deliberately. Keep parser state outside the callback, and call `stream.goToEnd()` once the target data has been found.
|
|
145
183
|
|
|
146
|
-
|
|
184
|
+
`Text` tokens preserve the source slice. They can contain whitespace-only formatting, and encoded references stay encoded.
|
|
147
185
|
|
|
148
|
-
|
|
186
|
+
## Selectors
|
|
149
187
|
|
|
150
|
-
|
|
188
|
+
Selectors use object paths instead of XPath strings:
|
|
151
189
|
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
190
|
+
```ts
|
|
191
|
+
const cookingBooks = [
|
|
192
|
+
{ axis: 'child', local: 'bookstore' },
|
|
193
|
+
{ axis: 'child', local: 'book', attributes: [{ local: 'category', value: 'COOKING' }] }
|
|
194
|
+
] as const;
|
|
195
|
+
```
|
|
157
196
|
|
|
158
|
-
|
|
197
|
+
Segments match by name, attributes, or plain text at a path depth. Use `axis: 'child'` for direct children and `axis: 'self-or-descendant'` for descendants.
|
|
159
198
|
|
|
160
|
-
|
|
199
|
+
Attribute and text segments currently do not combine with `local` or `prefix` on the same segment. Add a callback guard when that distinction matters.
|
|
161
200
|
|
|
162
|
-
|
|
201
|
+
## Object Helpers
|
|
163
202
|
|
|
164
|
-
|
|
203
|
+
`xmlToObject` returns a nested node tree with `local`, `prefix`, `attributes`, and `content`.
|
|
165
204
|
|
|
166
|
-
|
|
205
|
+
`xmlToSimplifiedObject` returns a more compact object shape where element names are stored under underscored keys such as `_book`.
|
|
167
206
|
|
|
168
|
-
|
|
169
|
-
| -------------- | ------------------------------- | ------------- | ------------- | -------------- | ------------------------------ |
|
|
170
|
-
| roxmltree:text | 67.12 | 14.33 | 83.29 | 80.08 | ±1.27% |
|
|
171
|
-
| roxmltree:byte | 12.48 | 78.65 | 16.45 | 14.90 | ±1.15% |
|
|
207
|
+
`tokensToXml`, `tokenToXml`, and `xmlToString` help rebuild markup from token or object data when you need a serialization step.
|
|
172
208
|
|
|
173
|
-
|
|
209
|
+
`htmlToMarkdown` converts simple HTML to Markdown and returns the string on `.string`:
|
|
174
210
|
|
|
175
|
-
|
|
211
|
+
```ts
|
|
212
|
+
const { string } = htmlToMarkdown('<h1>Title</h1><p>Hello <strong>world</strong>.</p>');
|
|
213
|
+
```
|
|
176
214
|
|
|
177
|
-
|
|
215
|
+
Object conversion omits whitespace-only text, comments, processing instructions, and entity declarations.
|
|
178
216
|
|
|
179
|
-
|
|
217
|
+
## Examples
|
|
180
218
|
|
|
181
|
-
|
|
219
|
+
- [Vanilla profiler](https://github.com/builder-group/community/tree/develop/examples/xml-tokenizer/vanilla/playground)
|
|
182
220
|
|
|
183
|
-
|
|
184
|
-
[xml-tokenizer] Total Time: 5345.0000 ms | Average Time per Run: 53.4500 ms | Median Time: 53.0000 ms | Runs: 100
|
|
185
|
-
[txml] Total Time: 395.0000 ms | Average Time per Run: 3.9500 ms | Median Time: 4.0000 ms | Runs: 100
|
|
186
|
-
[fast-xml-parser] Total Time: 1290.0000 ms | Average Time per Run: 12.9000 ms | Median Time: 13.0000 ms | Runs: 100
|
|
187
|
-
```
|
|
221
|
+
## FAQ
|
|
188
222
|
|
|
189
|
-
|
|
223
|
+
### Is this a DOM parser?
|
|
190
224
|
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
225
|
+
No. `xml-tokenizer` is built for streaming extraction and lightweight conversion. Use it when you want to inspect, select, or transform markup without committing to a full DOM model.
|
|
226
|
+
|
|
227
|
+
### Which config should I use for HTML?
|
|
228
|
+
|
|
229
|
+
Use `htmlConfig`. It enables HTML-oriented behavior such as raw text elements and implicit self-closing elements.
|
|
230
|
+
|
|
231
|
+
### Is `xmlConfig` full XML validation?
|
|
232
|
+
|
|
233
|
+
No. `xmlConfig` enforces document-level rules such as one root element and quoted attributes, but it does not validate schemas, process DTDs, or resolve entities. Use `{ ...xmlConfig, allowDtd: false }` to reject DTDs.
|
|
234
|
+
|
|
235
|
+
### Why does the tokenizer use callbacks instead of generators?
|
|
196
236
|
|
|
197
|
-
|
|
237
|
+
Callbacks keep the tokenizer loop direct and let the stream expose controls such as `goToEnd()`. Selectors and object helpers provide higher-level APIs when you need them.
|
|
198
238
|
|
|
199
|
-
|
|
239
|
+
### How does it compare to fast-xml-parser, txml, sax, and saxen?
|
|
200
240
|
|
|
201
|
-
-
|
|
202
|
-
- [txml](https://github.com/TobiasNickel/tXml)
|
|
203
|
-
- [roxmltree](https://github.com/RazrFalcon/roxmltree)
|
|
241
|
+
`xml-tokenizer` focuses on typed tokens plus small selector and object helpers. Use it when you want callback-based extraction with explicit parser state in TypeScript. Use a full XML object parser when you mainly need whole-document conversion.
|
|
@@ -1 +1 @@
|
|
|
1
|
-
import{ZERO as E,NINE as t,UPPERCASE_A as i,UPPERCASE_Z as r,LOWERCASE_A as e,LOWERCASE_Z as u,COLON as l,UNDERSCORE as R,
|
|
1
|
+
import{ZERO as E,NINE as t,UPPERCASE_A as i,UPPERCASE_Z as r,LOWERCASE_A as e,LOWERCASE_Z as u,COLON as l,UNDERSCORE as R,HYPHEN as m,PERIOD as s,SPACE as a,HORIZONTAL_TAB as o,LINE_FEED as C,CARRIAGE_RETURN as O}from"./ascii-constants.js";function c(n){return n>=E&&n<=t}function S(n){return n==null?!1:n<=128?n>=i&&n<=r||n>=e&&n<=u||n===l||n===R:n>=192&&n<=214||n>=216&&n<=246||n>=248&&n<=767||n>=880&&n<=893||n>=895&&n<=8191||n>=8204&&n<=8205||n>=8304&&n<=8591||n>=11264&&n<=12271||n>=12289&&n<=55295||n>=63744&&n<=64975||n>=65008&&n<=65533||n>=65536&&n<=983039}function P(n){return n==null?!1:n<=128?N(n):n===183||n>=192&&n<=214||n>=216&&n<=246||n>=248&&n<=767||n>=768&&n<=879||n>=880&&n<=893||n>=895&&n<=8191||n>=8204&&n<=8205||n>=8255&&n<=8256||n>=8304&&n<=8591||n>=11264&&n<=12271||n>=12289&&n<=55295||n>=63744&&n<=64975||n>=65008&&n<=65533||n>=65536&&n<=983039}function _(n){return n==null?!1:n<32?A(n):n!==65535&&n!==65534}function A(n){return n===a||n===o||n===C||n===O}function N(n){return n>=E&&n<=t||n>=i&&n<=r||n>=e&&n<=u||n===l||n===R||n===m||n===s}export{c as isAsciiDigit,_ as isXmlChar,P as isXmlName,N as isXmlNameByte,S as isXmlNameStart,A as isXmlSpaceByte};
|
package/package.json
CHANGED
|
@@ -1,9 +1,23 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "xml-tokenizer",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.46",
|
|
4
4
|
"private": false,
|
|
5
|
-
"description": "
|
|
6
|
-
"keywords": [
|
|
5
|
+
"description": "Streaming XML, HTML, and SVG tokenizer with typed tokens, selectors, and object helpers",
|
|
6
|
+
"keywords": [
|
|
7
|
+
"xml",
|
|
8
|
+
"html",
|
|
9
|
+
"svg",
|
|
10
|
+
"tokenizer",
|
|
11
|
+
"parser",
|
|
12
|
+
"xml-parser",
|
|
13
|
+
"html-parser",
|
|
14
|
+
"streaming-parser",
|
|
15
|
+
"sax",
|
|
16
|
+
"typescript",
|
|
17
|
+
"selector",
|
|
18
|
+
"object-parser",
|
|
19
|
+
"markup"
|
|
20
|
+
],
|
|
7
21
|
"homepage": "https://builder.group/?utm_source=package-json",
|
|
8
22
|
"bugs": {
|
|
9
23
|
"url": "https://github.com/builder-group/community/issues"
|
|
@@ -23,17 +37,16 @@
|
|
|
23
37
|
"README.md"
|
|
24
38
|
],
|
|
25
39
|
"devDependencies": {
|
|
26
|
-
"@types/node": "^25.0.8",
|
|
27
40
|
"@types/sax": "^1.2.7",
|
|
28
41
|
"@types/xml2js": "^0.4.14",
|
|
29
|
-
"camaro": "^6.2
|
|
30
|
-
"fast-xml-parser": "^5.
|
|
31
|
-
"sax": "^1.
|
|
42
|
+
"camaro": "^6.3.2",
|
|
43
|
+
"fast-xml-parser": "^5.8.0",
|
|
44
|
+
"sax": "^1.6.0",
|
|
32
45
|
"saxen": "^11.0.2",
|
|
33
|
-
"txml": "^
|
|
46
|
+
"txml": "^6.0.0",
|
|
34
47
|
"xml2js": "^0.6.2",
|
|
35
|
-
"@blgc/config": "0.0
|
|
36
|
-
"rollup-presets": "0.0.
|
|
48
|
+
"@blgc/config": "0.1.0",
|
|
49
|
+
"rollup-presets": "0.0.28"
|
|
37
50
|
},
|
|
38
51
|
"size-limit": [
|
|
39
52
|
{
|
|
@@ -46,7 +59,7 @@
|
|
|
46
59
|
"build:prod": "export NODE_ENV=production && pnpm build",
|
|
47
60
|
"clean": "shx rm -rf dist && shx rm -rf .turbo && shx rm -rf node_modules",
|
|
48
61
|
"install:clean": "pnpm run clean && pnpm install",
|
|
49
|
-
"lint": "eslint .
|
|
62
|
+
"lint": "eslint .",
|
|
50
63
|
"publish:patch": "pnpm build:prod && pnpm version patch && pnpm publish --no-git-checks --access=public",
|
|
51
64
|
"size": "size-limit --why",
|
|
52
65
|
"start:dev": "tsc -w",
|