xml-tokenizer 0.0.45 → 0.0.47
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +170 -132
- package/dist/cjs/config.js +1 -1
- package/dist/cjs/html-to-md.js +4 -4
- package/dist/cjs/tokenizer/XmlStream.js +1 -1
- package/dist/cjs/tokenizer/tokenize.js +1 -1
- package/dist/esm/config.js +1 -1
- package/dist/esm/html-to-md.js +6 -6
- package/dist/esm/tokenizer/XmlStream.js +1 -1
- package/dist/esm/tokenizer/tokenize.js +1 -1
- package/dist/types/config.d.ts +1 -0
- package/dist/types/tokenizer/XmlStream.d.ts +10 -3
- package/package.json +26 -12
package/README.md
CHANGED
|
@@ -17,187 +17,225 @@
|
|
|
17
17
|
</a>
|
|
18
18
|
</p>
|
|
19
19
|
|
|
20
|
-
|
|
20
|
+
`xml-tokenizer` is a callback-based XML, HTML, and SVG tokenizer for TypeScript. It emits typed tokens, supports early exit when you have the data you need, and includes small helpers for path selection and object conversion when a full DOM would be unnecessary.
|
|
21
21
|
|
|
22
|
-
|
|
23
|
-
|
|
22
|
+
- Emit `ElementStart`, `Attribute`, `Text`, `Cdata`, and other tokens without building a tree first
|
|
23
|
+
- Pick XML document mode, HTML, or SVG parsing behavior with `xmlConfig`, `htmlConfig`, and `svgConfig`
|
|
24
|
+
- Stop parsing from inside the callback with `stream.goToEnd()`
|
|
25
|
+
- Select matching token ranges with object-based path selectors
|
|
26
|
+
- Convert markup into nested or simplified objects when a tree shape is more convenient
|
|
24
27
|
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
28
|
+
```ts
|
|
29
|
+
import { htmlConfig, tokenize, type TXmlToken } from 'xml-tokenizer';
|
|
30
|
+
|
|
31
|
+
let title: string | null = null;
|
|
32
|
+
let insideTitle = false;
|
|
33
|
+
|
|
34
|
+
tokenize(
|
|
35
|
+
'<html><head><title>Hello</title></head></html>',
|
|
36
|
+
(token: TXmlToken, stream) => {
|
|
37
|
+
if (token.type === 'ElementStart' && token.local === 'title') {
|
|
38
|
+
insideTitle = true;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
if (insideTitle && token.type === 'Text') {
|
|
42
|
+
const text = token.text.trim();
|
|
43
|
+
if (text !== '') {
|
|
44
|
+
title = text;
|
|
45
|
+
stream.goToEnd();
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
},
|
|
49
|
+
htmlConfig
|
|
50
|
+
);
|
|
31
51
|
|
|
32
|
-
|
|
52
|
+
console.log(title); // Hello
|
|
53
|
+
```
|
|
33
54
|
|
|
34
|
-
|
|
55
|
+
## Install
|
|
35
56
|
|
|
36
|
-
|
|
57
|
+
```bash
|
|
58
|
+
npm install xml-tokenizer
|
|
59
|
+
```
|
|
37
60
|
|
|
38
|
-
|
|
61
|
+
## Usage
|
|
39
62
|
|
|
40
|
-
|
|
63
|
+
Use `tokenize` when you want to process markup as a stream:
|
|
41
64
|
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
65
|
+
```ts
|
|
66
|
+
import { tokenize, xmlConfig, type TXmlToken } from 'xml-tokenizer';
|
|
67
|
+
|
|
68
|
+
tokenize(
|
|
69
|
+
'<book id="1"><title>Dune</title></book>',
|
|
70
|
+
(token: TXmlToken) => {
|
|
71
|
+
switch (token.type) {
|
|
72
|
+
case 'ElementStart':
|
|
73
|
+
console.log('element', token.local);
|
|
74
|
+
break;
|
|
75
|
+
case 'Attribute':
|
|
76
|
+
console.log('attribute', token.local, token.value);
|
|
77
|
+
break;
|
|
78
|
+
case 'Text':
|
|
79
|
+
console.log('text', token.text.trim());
|
|
80
|
+
break;
|
|
81
|
+
}
|
|
82
|
+
},
|
|
83
|
+
xmlConfig
|
|
84
|
+
);
|
|
85
|
+
```
|
|
45
86
|
|
|
46
|
-
|
|
87
|
+
Use `select` when you only care about matching paths:
|
|
47
88
|
|
|
48
89
|
```ts
|
|
49
|
-
import { select,
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
// Or, parse XML to a stream of tokens
|
|
58
|
-
tokenize('<p>Hello World</p>', (token) => {
|
|
59
|
-
switch (token.type) {
|
|
60
|
-
case 'ElementStart':
|
|
61
|
-
console.log('Start of element:', token);
|
|
62
|
-
break;
|
|
63
|
-
case 'Text':
|
|
64
|
-
console.log('Text content:', token.text);
|
|
65
|
-
break;
|
|
66
|
-
// Handle other token types as needed
|
|
67
|
-
default:
|
|
68
|
-
console.log('Token:', token);
|
|
69
|
-
}
|
|
70
|
-
});
|
|
90
|
+
import { select, xmlConfig } from 'xml-tokenizer';
|
|
91
|
+
|
|
92
|
+
const xml = `
|
|
93
|
+
<bookstore>
|
|
94
|
+
<book category="COOKING"><title>Everyday Italian</title></book>
|
|
95
|
+
</bookstore>
|
|
96
|
+
`;
|
|
71
97
|
|
|
72
|
-
// Or, stream only a selection of tokens
|
|
73
98
|
select(
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
99
|
+
xml,
|
|
100
|
+
[
|
|
101
|
+
[
|
|
102
|
+
{ axis: 'child', local: 'bookstore' },
|
|
103
|
+
{ axis: 'child', local: 'book', attributes: [{ local: 'category', value: 'COOKING' }] }
|
|
104
|
+
]
|
|
105
|
+
],
|
|
106
|
+
(token, stream) => {
|
|
107
|
+
if (token.type === 'Text') {
|
|
108
|
+
console.log(token.text.trim());
|
|
109
|
+
stream.goToEnd();
|
|
110
|
+
}
|
|
111
|
+
},
|
|
112
|
+
xmlConfig
|
|
84
113
|
);
|
|
85
114
|
```
|
|
86
115
|
|
|
87
|
-
|
|
116
|
+
Use the object helpers when you want a small tree representation:
|
|
88
117
|
|
|
89
|
-
|
|
118
|
+
```ts
|
|
119
|
+
import { htmlToMarkdown, xmlToObject, xmlToSimplifiedObject } from 'xml-tokenizer';
|
|
90
120
|
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
- **ElementStart**: `<ns:elem`
|
|
95
|
-
- **Attribute**: `ns:attr="value"`
|
|
96
|
-
- **ElementEnd**:
|
|
97
|
-
- Open: `>`
|
|
98
|
-
- Close: `</ns:name>`
|
|
99
|
-
- Empty: `/>`
|
|
100
|
-
- **Text**: Text content between elements, including whitespace.
|
|
101
|
-
- **Cdata**: `<![CDATA[text]]>`
|
|
121
|
+
const tree = xmlToObject('<book id="1"><title>Dune</title></book>');
|
|
122
|
+
const simplified = xmlToSimplifiedObject('<book id="1"><title>Dune</title></book>');
|
|
123
|
+
```
|
|
102
124
|
|
|
103
|
-
|
|
125
|
+
`tree` contains nested nodes:
|
|
104
126
|
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
127
|
+
```ts
|
|
128
|
+
const tree = {
|
|
129
|
+
local: 'book',
|
|
130
|
+
attributes: [{ local: 'id', value: '1' }],
|
|
131
|
+
content: [{ local: 'title', attributes: [], content: ['Dune'] }]
|
|
132
|
+
};
|
|
133
|
+
```
|
|
109
134
|
|
|
110
|
-
|
|
135
|
+
`simplified` stores element names under underscored keys:
|
|
111
136
|
|
|
112
|
-
|
|
137
|
+
```ts
|
|
138
|
+
const simplified = {
|
|
139
|
+
_book: [
|
|
140
|
+
{
|
|
141
|
+
attributes: { id: '1' },
|
|
142
|
+
_title: [{ text: 'Dune' }]
|
|
143
|
+
}
|
|
144
|
+
]
|
|
145
|
+
};
|
|
146
|
+
```
|
|
113
147
|
|
|
114
|
-
|
|
148
|
+
## Configs
|
|
115
149
|
|
|
116
|
-
|
|
117
|
-
| -------------------- | ------------------------------- | ------------- | ------------- | -------------- | ------------------------------ |
|
|
118
|
-
| xml-tokenizer | 46.87 | 19.47 | 24.57 | 21.33 | ±2.06% |
|
|
119
|
-
| xml-tokenizer (dist) | 53.70 | 17.31 | 25.20 | 18.62 | ±3.28% |
|
|
120
|
-
| xml-tokenizer (npm) | 163.00 | 5.03 | 8.50 | 6.13 | ±2.32% |
|
|
121
|
-
| fast-xml-parser | 66.00 | 14.01 | 20.73 | 15.15 | ±3.34% |
|
|
122
|
-
| txml | 234.52 | 3.38 | 7.61 | 4.26 | ±4.00% |
|
|
123
|
-
| xml2js | 36.21 | 25.58 | 37.28 | 27.61 | ±4.39% |
|
|
150
|
+
Choose the config that matches the input:
|
|
124
151
|
|
|
125
|
-
|
|
152
|
+
| Config | Use for |
|
|
153
|
+
| ------------ | ------------------------------------ |
|
|
154
|
+
| `xmlConfig` | XML document mode |
|
|
155
|
+
| `htmlConfig` | HTML with raw text and void elements |
|
|
156
|
+
| `svgConfig` | SVG fragments and documents |
|
|
126
157
|
|
|
127
|
-
|
|
128
|
-
| ------------------- | ------------------------------- | ------------- | ------------- | -------------- | ------------------------------ |
|
|
129
|
-
| xml-tokenizer | 53.03 | 18.30 | 19.45 | 18.86 | ±0.81% |
|
|
130
|
-
| xml-tokenizer (npm) | 166.61 | 5.62 | 7.16 | 6.00 | ±0.88% |
|
|
131
|
-
| saxen | 500.99 | 1.83 | 4.79 | 2.00 | ±1.52% |
|
|
132
|
-
| sax | 64.44 | 14.96 | 16.34 | 15.52 | ±0.67% |
|
|
158
|
+
All configs are `TXmlStreamOptions`, so you can pass custom options when you need different parser behavior:
|
|
133
159
|
|
|
134
|
-
|
|
160
|
+
```ts
|
|
161
|
+
tokenize(markup, onToken, {
|
|
162
|
+
...htmlConfig,
|
|
163
|
+
contextSliceSize: 80
|
|
164
|
+
});
|
|
165
|
+
```
|
|
135
166
|
|
|
136
|
-
|
|
167
|
+
## Tokens
|
|
137
168
|
|
|
138
|
-
|
|
139
|
-
pnpm run bench
|
|
140
|
-
```
|
|
169
|
+
`tokenize` can emit these token types:
|
|
141
170
|
|
|
142
|
-
|
|
171
|
+
| Token | Example |
|
|
172
|
+
| ----------------------- | ------------------------ |
|
|
173
|
+
| `ProcessingInstruction` | `<?target content?>` |
|
|
174
|
+
| `Comment` | `<!-- text -->` |
|
|
175
|
+
| `EntityDeclaration` | `<!ENTITY name "value">` |
|
|
176
|
+
| `ElementStart` | `<book` |
|
|
177
|
+
| `Attribute` | `id="1"` |
|
|
178
|
+
| `ElementEnd` | `>`, `</book>`, or `/>` |
|
|
179
|
+
| `Text` | Text between elements |
|
|
180
|
+
| `Cdata` | `<![CDATA[text]]>` |
|
|
143
181
|
|
|
144
|
-
|
|
182
|
+
Structure-sensitive code should handle `ElementStart`, `ElementEnd`, and `Attribute` deliberately. Keep parser state outside the callback, and call `stream.goToEnd()` once the target data has been found.
|
|
145
183
|
|
|
146
|
-
|
|
184
|
+
`Text` tokens preserve the source slice. They can contain whitespace-only formatting, and encoded references stay encoded.
|
|
147
185
|
|
|
148
|
-
|
|
186
|
+
## Selectors
|
|
149
187
|
|
|
150
|
-
|
|
188
|
+
Selectors use object paths instead of XPath strings:
|
|
151
189
|
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
190
|
+
```ts
|
|
191
|
+
const cookingBooks = [
|
|
192
|
+
{ axis: 'child', local: 'bookstore' },
|
|
193
|
+
{ axis: 'child', local: 'book', attributes: [{ local: 'category', value: 'COOKING' }] }
|
|
194
|
+
] as const;
|
|
195
|
+
```
|
|
157
196
|
|
|
158
|
-
|
|
197
|
+
Segments match by name, attributes, or plain text at a path depth. Use `axis: 'child'` for direct children and `axis: 'self-or-descendant'` for descendants.
|
|
159
198
|
|
|
160
|
-
|
|
199
|
+
Attribute and text segments currently do not combine with `local` or `prefix` on the same segment. Add a callback guard when that distinction matters.
|
|
161
200
|
|
|
162
|
-
|
|
201
|
+
## Object Helpers
|
|
163
202
|
|
|
164
|
-
|
|
203
|
+
`xmlToObject` returns a nested node tree with `local`, `prefix`, `attributes`, and `content`.
|
|
165
204
|
|
|
166
|
-
|
|
205
|
+
`xmlToSimplifiedObject` returns a more compact object shape where element names are stored under underscored keys such as `_book`.
|
|
167
206
|
|
|
168
|
-
|
|
169
|
-
| -------------- | ------------------------------- | ------------- | ------------- | -------------- | ------------------------------ |
|
|
170
|
-
| roxmltree:text | 67.12 | 14.33 | 83.29 | 80.08 | ±1.27% |
|
|
171
|
-
| roxmltree:byte | 12.48 | 78.65 | 16.45 | 14.90 | ±1.15% |
|
|
207
|
+
`tokensToXml`, `tokenToXml`, and `xmlToString` help rebuild markup from token or object data when you need a serialization step.
|
|
172
208
|
|
|
173
|
-
|
|
209
|
+
`htmlToMarkdown` converts simple HTML to Markdown and returns the string on `.string`:
|
|
174
210
|
|
|
175
|
-
|
|
211
|
+
```ts
|
|
212
|
+
const { string } = htmlToMarkdown('<h1>Title</h1><p>Hello <strong>world</strong>.</p>');
|
|
213
|
+
```
|
|
176
214
|
|
|
177
|
-
|
|
215
|
+
Object conversion omits whitespace-only text, comments, processing instructions, and entity declarations.
|
|
178
216
|
|
|
179
|
-
|
|
217
|
+
## Examples
|
|
180
218
|
|
|
181
|
-
|
|
219
|
+
- [Vanilla profiler](https://github.com/builder-group/community/tree/develop/examples/xml-tokenizer/vanilla/playground)
|
|
182
220
|
|
|
183
|
-
|
|
184
|
-
[xml-tokenizer] Total Time: 5345.0000 ms | Average Time per Run: 53.4500 ms | Median Time: 53.0000 ms | Runs: 100
|
|
185
|
-
[txml] Total Time: 395.0000 ms | Average Time per Run: 3.9500 ms | Median Time: 4.0000 ms | Runs: 100
|
|
186
|
-
[fast-xml-parser] Total Time: 1290.0000 ms | Average Time per Run: 12.9000 ms | Median Time: 13.0000 ms | Runs: 100
|
|
187
|
-
```
|
|
221
|
+
## FAQ
|
|
188
222
|
|
|
189
|
-
|
|
223
|
+
### Is this a DOM parser?
|
|
190
224
|
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
225
|
+
No. `xml-tokenizer` is built for streaming extraction and lightweight conversion. Use it when you want to inspect, select, or transform markup without committing to a full DOM model.
|
|
226
|
+
|
|
227
|
+
### Which config should I use for HTML?
|
|
228
|
+
|
|
229
|
+
Use `htmlConfig`. It enables HTML-oriented behavior such as raw text elements and implicit self-closing elements.
|
|
230
|
+
|
|
231
|
+
### Is `xmlConfig` full XML validation?
|
|
232
|
+
|
|
233
|
+
No. `xmlConfig` enforces document-level rules such as one root element and quoted attributes, but it does not validate schemas, process DTDs, or resolve entities. Use `{ ...xmlConfig, allowDtd: false }` to reject DTDs.
|
|
234
|
+
|
|
235
|
+
### Why does the tokenizer use callbacks instead of generators?
|
|
196
236
|
|
|
197
|
-
|
|
237
|
+
Callbacks keep the tokenizer loop direct and let the stream expose controls such as `goToEnd()`. Selectors and object helpers provide higher-level APIs when you need them.
|
|
198
238
|
|
|
199
|
-
|
|
239
|
+
### How does it compare to fast-xml-parser, txml, sax, and saxen?
|
|
200
240
|
|
|
201
|
-
-
|
|
202
|
-
- [txml](https://github.com/TobiasNickel/tXml)
|
|
203
|
-
- [roxmltree](https://github.com/RazrFalcon/roxmltree)
|
|
241
|
+
`xml-tokenizer` focuses on typed tokens plus small selector and object helpers. Use it when you want callback-based extraction with explicit parser state in TypeScript. Use a full XML object parser when you mainly need whole-document conversion.
|
package/dist/cjs/config.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
"use strict";const
|
|
1
|
+
"use strict";const e={strictDocument:!0,lowercaseNames:!1,allowDtd:!0,rawTextElements:null,implicitSelfClosingElements:null,contextSliceSize:30},t={strictDocument:!1,lowercaseNames:!0,allowDtd:!0,rawTextElements:["script","style","title","textarea"],implicitSelfClosingElements:["area","base","br","col","embed","hr","img","input","link","meta","param","source","track","wbr"],contextSliceSize:30},l={strictDocument:!1,lowercaseNames:!1,allowDtd:!0,rawTextElements:["style"],implicitSelfClosingElements:[],contextSliceSize:30};exports.htmlConfig=t,exports.svgConfig=l,exports.xmlConfig=e;
|
package/dist/cjs/html-to-md.js
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
"use strict";var b=require("./config.js"),
|
|
2
|
-
`},blockquote:(r,t)=>{const n=
|
|
3
|
-
`,code:r=>{const t=
|
|
1
|
+
"use strict";var b=require("./config.js"),c=require("./xml-to-string.js"),C=Object.defineProperty,a=Object.getOwnPropertySymbols,m=Object.prototype.hasOwnProperty,d=Object.prototype.propertyIsEnumerable,f=(e,l,o)=>l in e?C(e,l,{enumerable:!0,configurable:!0,writable:!0,value:o}):e[l]=o,$=(e,l)=>{for(var o in l||(l={}))m.call(l,o)&&f(e,o,l[o]);if(a)for(var o of a(l))d.call(l,o)&&f(e,o,l[o]);return e},X=(e,l)=>{var o={};for(var i in e)m.call(e,i)&&l.indexOf(i)<0&&(o[i]=e[i]);if(e!=null&&a)for(var i of a(e))l.indexOf(i)<0&&d.call(e,i)&&(o[i]=e[i]);return o};function v(e,l={}){const o=l,{xmlOptions:i=b.htmlConfig,transformers:h,skipNodes:p=y}=o,S=X(o,["xmlOptions","transformers","skipNodes"]);return c.xmlToString(e,$({xmlOptions:i,skipNodes:p,transformers:$({h1:(r,t)=>{const n=u(c.getXmlStringNodeContent(r));return n===""?"":`${s(t)}# ${n}`},h2:(r,t)=>{const n=u(c.getXmlStringNodeContent(r));return n===""?"":`${s(t)}## ${n}`},h3:(r,t)=>{const n=u(c.getXmlStringNodeContent(r));return n===""?"":`${s(t)}### ${n}`},h4:(r,t)=>{const n=u(c.getXmlStringNodeContent(r));return n===""?"":`${s(t)}#### ${n}`},h5:(r,t)=>{const n=u(c.getXmlStringNodeContent(r));return n===""?"":`${s(t)}##### ${n}`},h6:(r,t)=>{const n=u(c.getXmlStringNodeContent(r));return n===""?"":`${s(t)}###### ${n}`},p:(r,t)=>{const n=u(c.getXmlStringNodeContent(r));return n===""?"":`${s(t)}${n}`},strong:r=>{const t=u(c.getXmlStringNodeContent(r));return t===""?"":`**${t}**`},b:r=>{const t=u(c.getXmlStringNodeContent(r));return t===""?"":`**${t}**`},em:r=>{const t=u(c.getXmlStringNodeContent(r));return t===""?"":`*${t}*`},i:r=>{const t=u(c.getXmlStringNodeContent(r));return t===""?"":`*${t}*`},a:r=>{var t;const n=(t=r.attributes.find(N=>N.local==="href"))==null?void 0:t.value,g=u(c.getXmlStringNodeContent(r));return g===""?"":n!=null?`[${g}](${n})`:g},ul:(r,t)=>{const n=c.getXmlStringNodeContent(r);return n===""?"":`${s(t)}${n}`},ol:(r,t)=>{const n=c.getXmlStringNodeContent(r);return n===""?"":`${s(t)}${n}`},li:(r,t)=>{const n=u(c.getXmlStringNodeContent(r));if(n==="")return"";const g=O(t);return`${" ".repeat(Math.max(0,g-1))}- ${n}
|
|
2
|
+
`},blockquote:(r,t)=>{const n=u(c.getXmlStringNodeContent(r));return n===""?"":`${s(t)}> ${n}`},hr:(r,t)=>`${s(t)}---`,br:()=>`
|
|
3
|
+
`,code:r=>{const t=u(c.getXmlStringNodeContent(r));return t===""?"":`\`${t}\``},pre:(r,t)=>{const n=c.getXmlStringNodeContent(r);return n===""?"":`${s(t)}\`\`\`
|
|
4
4
|
${n}
|
|
5
|
-
\`\`\``}},h)},S))}function
|
|
5
|
+
\`\`\``}},h)},S))}function u(e){return e.replace(/\s+/g," ").trim()}function s(e){if(e.length===0)return"";const l=e[e.length-1];if(l==null||l.content.length===0)return"";for(let o=l.content.length-1;o>=0;o--){const i=l.content[o];if(typeof i=="object"&&i.string!=null&&i.string!==""){if(k.has(i.local))return`
|
|
6
6
|
|
|
7
7
|
`;if(x.has(i.local))return`
|
|
8
8
|
`}}return""}function O(e){let l=0;for(let o=e.length-1;o>=0;o--){const i=e[o];i&&(i.local==="ul"||i.local==="ol")&&l++}return l}const k=new Set(["h1","h2","h3","h4","h5","h6","p","div","blockquote","hr","pre","ul","ol","li"]),x=new Set(["strong","b","em","i","a","code","br"]),y=["script","style","meta","link","noscript","iframe","object","embed","param","base","title","head"];exports.htmlToMarkdown=v;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
"use strict";var
|
|
1
|
+
"use strict";var o=require("./ascii-constants.js"),l=require("./utils.js"),n=require("./XmlError.js"),g=Object.defineProperty,d=Object.getOwnPropertySymbols,x=Object.prototype.hasOwnProperty,E=Object.prototype.propertyIsEnumerable,u=(a,t,e)=>t in a?g(a,t,{enumerable:!0,configurable:!0,writable:!0,value:e}):a[t]=e,_=(a,t)=>{for(var e in t||(t={}))x.call(t,e)&&u(a,e,t[e]);if(d)for(var e of d(t))E.call(t,e)&&u(a,e,t[e]);return a},p=(a,t,e)=>u(a,typeof t!="symbol"?t+"":t,e);class m{constructor(t,e={}){p(this,"_text"),p(this,"_pos"),p(this,"_end"),p(this,"config");const{pos:s=0,strictDocument:r=!0,lowercaseNames:i=!1,allowDtd:h=!0,rawTextElements:c=null,implicitSelfClosingElements:C=null,contextSliceSize:f=!1}=e;this._text=t,this._pos=s,this._end=this._text.length,this.config={strictDocument:r,lowercaseNames:i,allowDtd:h,rawTextElements:c,implicitSelfClosingElements:C,contextSliceSize:f}}clone(){return new m(this._text,_({pos:this._pos},this.config))}getPos(){return this._pos}atEnd(){return this._pos>=this._end}currCodeUnit(){if(this._pos>=this._end)throw new n.XmlError({type:"UnexpectedEndOfStream"});return this._text.charCodeAt(this._pos)}currCodeUnitUnchecked(){return this._text.charCodeAt(this._pos)}nextCodeUnit(){if(this._pos+1>=this._end)throw new n.XmlError({type:"UnexpectedEndOfStream"});return this._text.charCodeAt(this._pos+1)}consumeCodeUnit(t){const e=this.currCodeUnit();if(e!==t)throw new n.XmlError({type:"InvalidChar",expected:t,actual:e},this.genTextPos());this._pos+=1}tryConsumeCodeUnit(t){return this.currCodeUnit()===t?(this._pos+=1,!0):!1}consumeCodeUnitsWhile(t){const e=this._pos;return this.skipCodeUnitsWhile(t),this.sliceBack(e)}skipCodeUnitsWhile(t){for(;this._pos<this._end&&t(this.currCodeUnitUnchecked(),this);){if(!l.isXmlChar(this.currCodeUnitUnchecked()))throw new n.XmlError({type:"NonXmlChar",char:String.fromCodePoint(this.currCodeUnitUnchecked())},this.genTextPos());this._pos+=1}}advance(t){this._pos+=t}goTo(t){this._pos=t}goToEnd(){this._pos=this._end}startsWith(t){return this._text.startsWith(t,this._pos)}startsWithIgnoreCase(t){if(this._pos+t.length>this._end)return!1;for(let e=0;e<t.length;e++){const s=this._text.charCodeAt(this._pos+e),r=t.charCodeAt(e),i=s>=97&&s<=122?s-32:s,h=r>=97&&r<=122?r-32:r;if(i!==h)return!1}return!0}skipString(t){if(!this.startsWith(t))throw new n.XmlError({type:"InvalidString",expected:t},this.genTextPos());this._pos+=t.length}sliceBack(t){return this._text.slice(t,this._pos)}rangeFrom(t){return{start:t,end:this._pos}}skipSpaces(){for(;this.startsWithSpace();)this._pos+=1}startsWithSpace(){return this._pos<this._end&&l.isXmlSpaceByte(this.currCodeUnitUnchecked())}consumeSpaces(){if(this._pos>=this._end)throw new n.XmlError({type:"UnexpectedEndOfStream"},this.genTextPos());if(!this.startsWithSpace())throw new n.XmlError({type:"InvalidChar",expected:"a whitespace",actual:this.currCodeUnitUnchecked()},this.genTextPos());this.skipSpaces()}tryConsumeReference(){const t=this._pos,e=this.clone(),s=e.consumeReference();return s!=null?(this._pos+=e.getPos()-t,s):null}consumeReference(){if(!this.tryConsumeCodeUnit(o.AMPERSAND))return null;let t;if(this.tryConsumeCodeUnit(o.HASH)){let e,s;this.tryConsumeCodeUnit(o.LOWERCASE_X)?(e=this.consumeCodeUnitsWhile(i=>i>=o.ZERO&&i<=o.NINE||i>=o.UPPERCASE_A&&i<=o.UPPERCASE_F||i>=o.LOWERCASE_A&&i<=o.LOWERCASE_F),s=16):(e=this.consumeCodeUnitsWhile(i=>l.isAsciiDigit(i)),s=10);const r=parseInt(e,s);isNaN(r)||!l.isXmlChar(r)?t=null:t={type:"Char",value:String.fromCodePoint(r)}}else{const e=this.consumeName();switch(e){case"quot":t={type:"Char",value:'"'};break;case"amp":t={type:"Char",value:"&"};break;case"apos":t={type:"Char",value:"'"};break;case"lt":t={type:"Char",value:"<"};break;case"gt":t={type:"Char",value:">"};break;default:t={type:"Entity",value:e}}}return this.tryConsumeCodeUnit(o.SEMICOLON)?t:null}consumeName(){const t=this._pos;this.skipName();const e=this.sliceBack(t);if(e.length===0)throw new n.XmlError({type:"InvalidName"},this.genTextPosFrom(t));return e}skipName(){const t=this._pos;for(;this._pos<this._end;){const e=this.currCodeUnitUnchecked();if(this._pos===t){if(!l.isXmlNameStart(e))throw new n.XmlError({type:"InvalidName"},this.genTextPosFrom(t))}else if(!l.isXmlName(e))break;this._pos+=1}}consumeQName(){var t,e;const s=this._pos;let r=null;for(;this._pos<this._end;){const c=this.currCodeUnitUnchecked();if(c===o.COLON)if(r==null)r=this._pos,this._pos+=1;else throw new n.XmlError({type:"InvalidName"},this.genTextPosFrom(s));else if(l.isXmlName(c))this._pos+=1;else break}let i,h;if(r!=null?(i=this._text.slice(s,r),h=this.sliceBack(r+1)):(i="",h=this.sliceBack(s)),i.length>0&&!l.isXmlNameStart((t=i[0])==null?void 0:t.codePointAt(0)))throw new n.XmlError({type:"InvalidName"},this.genTextPosFrom(s));if(h.length>0){if(!l.isXmlNameStart((e=h[0])==null?void 0:e.codePointAt(0)))throw new n.XmlError({type:"InvalidName"},this.genTextPosFrom(s))}else throw new n.XmlError({type:"InvalidName"},this.genTextPosFrom(s));return this.config.lowercaseNames&&(i=i.replace(/[A-Z]/g,c=>c.toLowerCase()),h=h.replace(/[A-Z]/g,c=>c.toLowerCase())),[i,h]}consumeQuote(){const t=this.currCodeUnit();if(t===o.SINGLE_QUOTE||t===o.DOUBLE_QUOTE)return this._pos+=1,t;throw new n.XmlError({type:"InvalidChar",expected:"a quote",actual:t},this.genTextPos())}getTextAround(t,e){const s=Math.floor(e/2),r=Math.max(0,t-s),i=Math.min(this._end,t+s+e%2);return this._text.slice(r,i)}genTextPos(){return this.genTextPosFrom(this._pos)}genTextPosFrom(t){const e=Math.min(t,this._end);let s=1,r=1;for(let i=0;i<e;i++)this._text.charCodeAt(i)===o.LINE_FEED?(s++,r=1):r++;return _({row:s,col:r},this.config.contextSliceSize?{contextSlice:this.getTextAround(t,this.config.contextSliceSize)}:{})}}exports.XmlStream=m;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
"use strict";var U=require("../get-q-name.js"),s=require("./ascii-constants.js"),
|
|
1
|
+
"use strict";var U=require("../get-q-name.js"),s=require("./ascii-constants.js"),H=require("./utils.js"),c=require("./XmlError.js"),Q=require("./XmlStream.js");const D="\uFEFF",A="<?xml ",k="?>",F="<!ENTITY",O="<!ELEMENT",G="<!ATTLIST",b="<!NOTATION",p="<!DOCTYPE",w="<?",W="?>",E="<!--",y="-->",B="<![CDATA[",f="]]>",M="NDATA",N="PUBLIC",q="SYSTEM",K="version",Y="encoding",z="standalone";function $(t,n,e={}){P(new Q.XmlStream(t,e),n)}function P(t,n){if(t.startsWith(D)&&t.advance(1),t.startsWith(A)&&j(t),m(t,n),t.config.strictDocument){if(t.skipSpaces(),t.startsWithIgnoreCase(p)&&(S(t,n),m(t,n)),t.skipSpaces(),!t.atEnd()&&t.currCodeUnit()===s.LESS_THAN&&T(t,n),m(t,n),!t.atEnd())throw new c.XmlError({type:"UnknownToken",message:"Not at end"},t.genTextPos())}else for(;!t.atEnd();)t.currCodeUnit()===s.LESS_THAN?T(t,n):R(t,n),m(t,n)}function m(t,n){for(;!t.atEnd();)if(t.skipSpaces(),t.startsWith(E))g(t,n);else if(!t.config.strictDocument&&t.startsWithIgnoreCase(p))S(t,n);else if(t.startsWith(w))h(t,n);else break}function j(t){function n(e){if(e.startsWithSpace())e.skipSpaces();else if(!e.startsWith(k)&&!e.atEnd())throw new c.XmlError({type:"InvalidChar",expected:"a whitespace",actual:e.currCodeUnitUnchecked()},e.genTextPos())}if(t.advance(5),n(t),!t.startsWith(K))throw new c.XmlError({type:"InvalidString",expected:"version"},t.genTextPos());d(t),n(t),t.startsWith(Y)&&(d(t),n(t)),t.startsWith(z)&&d(t),t.skipSpaces(),t.skipString(k)}function g(t,n){const e=t.getPos();t.advance(4);const o=t.consumeCodeUnitsWhile((r,i)=>!(r===s.HYPHEN&&i.startsWith(y)));if(t.skipString(y),o.includes("--")||o.endsWith("-"))throw new c.XmlError({type:"InvalidComment"},t.genTextPosFrom(e));n({type:"Comment",text:o,range:t.rangeFrom(e)},t)}function h(t,n){if(t.startsWith(A))throw new c.XmlError({type:"UnexpectedDeclaration"},t.genTextPos());const e=t.getPos();t.advance(2);const o=t.consumeName();t.skipSpaces();const r=t.consumeCodeUnitsWhile((i,a)=>!(i===s.QUESTION_MARK&&a.startsWith(W)));t.skipString(W),n({type:"ProcessingInstruction",target:o,content:r.length===0?void 0:r,range:t.rangeFrom(e)},t)}function S(t,n){if(!t.config.allowDtd)throw new c.XmlError({type:"DtdDetected"});const e=t.getPos();if(J(t),t.skipSpaces(),t.currCodeUnit()===s.GREATER_THAN){t.advance(1);return}for(t.advance(1);!t.atEnd();)if(t.skipSpaces(),t.startsWith(F))Z(t,n);else if(t.startsWith(E))g(t,n);else if(t.startsWith(w))h(t,n);else if(t.startsWith("]"))if(t.advance(1),t.skipSpaces(),t.currCodeUnit()===s.GREATER_THAN){t.advance(1);break}else throw new c.XmlError({type:"InvalidChar",expected:"'>'",actual:t.currCodeUnitUnchecked()},t.genTextPos());else if(t.startsWith(O)||t.startsWith(G)||t.startsWith(b))try{tt(t)}catch(o){throw new c.XmlError({type:"UnknownToken",message:"Failed to consume declaration"},t.genTextPosFrom(e))}else throw new c.XmlError({type:"UnknownToken",message:"Failed to parse doctype"},t.genTextPos())}function J(t){t.advance(9),t.consumeSpaces(),t.skipName(),t.skipSpaces(),x(t),t.skipSpaces();const n=t.currCodeUnit();if(n!==s.OPEN_SQUARE_BRACKET&&n!==s.GREATER_THAN)throw new c.XmlError({type:"InvalidChar",expected:"'[' or '>'",actual:n},t.genTextPos())}function x(t){if(t.startsWith(q)||t.startsWith(N)){const n=t.getPos();t.advance(6);const e=t.sliceBack(n);t.consumeSpaces();const o=t.consumeQuote();if(t.consumeCodeUnitsWhile(r=>r!==o),t.consumeCodeUnit(o),e===N){t.consumeSpaces();const r=t.consumeQuote();t.consumeCodeUnitsWhile(i=>i!==r),t.consumeCodeUnit(r)}return!0}return!1}function Z(t,n){t.advance(8),t.consumeSpaces();const e=!t.tryConsumeCodeUnit(s.PERCENT);e||t.consumeSpaces();const o=t.consumeName();t.consumeSpaces();const r=V(t,e);r!==null&&n({type:"EntityDeclaration",name:o,definition:r},t),t.skipSpaces(),t.consumeCodeUnit(s.GREATER_THAN)}function V(t,n){const e=t.currCodeUnit();if(e===s.DOUBLE_QUOTE||e===s.SINGLE_QUOTE){const o=t.consumeQuote(),r=t.getPos();t.skipCodeUnitsWhile(a=>a!==o);const i=t.sliceBack(r);return t.consumeCodeUnit(o),i}else if(e===s.UPPERCASE_S||e===s.UPPERCASE_P){if(x(t))return n&&(t.skipSpaces(),t.startsWith(M)&&(t.advance(5),t.consumeSpaces(),t.skipName())),null;throw new c.XmlError({type:"InvalidExternalID"},t.genTextPos())}else throw new c.XmlError({type:"InvalidChar",expected:"a quote, SYSTEM or PUBLIC",actual:e},t.genTextPos())}function tt(t){t.skipCodeUnitsWhile(n=>n!==s.GREATER_THAN),t.consumeCodeUnit(s.GREATER_THAN)}function T(t,n){const e=t.getPos();t.advance(1);const[o,r]=t.consumeQName();n({type:"ElementStart",prefix:o,local:r,start:e},t);let i=!1;for(;!t.atEnd();){const a=t.startsWithSpace();t.skipSpaces();const l=t.getPos(),C=t.currCodeUnit();if(C===s.SLASH){t.advance(1),t.consumeCodeUnit(s.GREATER_THAN);const u=t.rangeFrom(l);n({type:"ElementEnd",end:{type:"Empty"},range:u},t);break}else if(C===s.GREATER_THAN){t.advance(1);const u=t.rangeFrom(l);if(t.config.implicitSelfClosingElements!=null&&t.config.implicitSelfClosingElements.includes(U.getQName(r,o))){n({type:"ElementEnd",end:{type:"Empty"},range:u},t);break}else{n({type:"ElementEnd",end:{type:"Open"},range:u},t),i=!0;break}}else{if(!a)throw new c.XmlError({type:"InvalidChar",expected:"a whitespace",actual:t.currCodeUnitUnchecked()},t.genTextPos());const[u,I,X]=d(t),L=t.getPos();n({type:"Attribute",range:{start:l,end:L},prefix:u,local:I,value:X},t)}}if(i){if(t.config.rawTextElements!=null){const a=U.getQName(r,o);if(t.config.rawTextElements.includes(a)){v(t,n,`</${a}>`);return}}v(t,n)}}function d(t){const[n,e]=t.consumeQName();let o;const r=t.getPos();if(t.skipSpaces(),t.tryConsumeCodeUnit(s.EQUALS)){t.skipSpaces();const i=t.currCodeUnit();if(i===s.SINGLE_QUOTE||i===s.DOUBLE_QUOTE){const a=t.consumeQuote();o=t.consumeCodeUnitsWhile(l=>l!==a&&l!==s.LESS_THAN),t.consumeCodeUnit(a)}else if(!t.config.strictDocument)o=t.consumeCodeUnitsWhile(a=>!H.isXmlSpaceByte(a)&&a!==s.GREATER_THAN&&a!==s.SLASH&&a!==s.LESS_THAN);else throw new c.XmlError({type:"InvalidChar",expected:"a quote",actual:i},t.genTextPos())}else if(!t.config.strictDocument)t.goTo(r),o="true";else throw new c.XmlError({type:"InvalidChar",expected:s.EQUALS,actual:t.currCodeUnit()},t.genTextPos());return[n,e,o]}function v(t,n,e=s.LESS_THAN){for(;!t.atEnd();)if(_(t,e)){const o=t.nextCodeUnit();if(o===s.EXCLAMATION_MARK)if(t.startsWith(E))g(t,n);else if(t.startsWith(B))nt(t,n);else if(!t.config.strictDocument&&t.startsWithIgnoreCase(p))S(t,n);else throw new c.XmlError({type:"UnknownToken",message:"Failed to parse content"},t.genTextPos());else if(o===s.QUESTION_MARK)h(t,n);else if(o===s.SLASH){et(t,n);break}else T(t,n)}else R(t,n,e)}function nt(t,n){const e=t.getPos();t.advance(9);const o=t.consumeCodeUnitsWhile((i,a)=>!(i===s.CLOSE_SQUARE_BRACKET&&a.startsWith(f)));t.skipString(f);const r=t.rangeFrom(e);n({type:"Cdata",text:o,range:r},t)}function et(t,n){const e=t.getPos();t.advance(2);const[o,r]=t.consumeQName();t.skipSpaces(),t.consumeCodeUnit(s.GREATER_THAN);const i=t.rangeFrom(e);n({type:"ElementEnd",end:{type:"Close",prefix:o,local:r},range:i},t)}function R(t,n,e=s.LESS_THAN){const o=t.getPos(),r=typeof e=="number"?t.consumeCodeUnitsWhile(i=>i!==e):t.consumeCodeUnitsWhile((i,a)=>!_(a,e));if(t.config.strictDocument&&r.includes(f))throw new c.XmlError({type:"InvalidCharacterData"},t.genTextPos());n({type:"Text",text:r,range:t.rangeFrom(o)},t)}function _(t,n){return typeof n=="number"?t.currCodeUnit()===n:t.config.lowercaseNames?t.startsWithIgnoreCase(n):t.startsWith(n)}exports.tokenize=$,exports.tokenizeXmlStream=P;
|
package/dist/esm/config.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
const e={strictDocument:!0,allowDtd:!0,rawTextElements:null,implicitSelfClosingElements:null,contextSliceSize:30},t={strictDocument:!1,allowDtd:!0,rawTextElements:["script","style","title","textarea"],implicitSelfClosingElements:["area","base","br","col","embed","hr","img","input","link","meta","param","source","track","wbr"],contextSliceSize:30},l={strictDocument:!1,allowDtd:!0,rawTextElements:["style"],implicitSelfClosingElements:[],contextSliceSize:30};export{t as htmlConfig,l as svgConfig,e as xmlConfig};
|
|
1
|
+
const e={strictDocument:!0,lowercaseNames:!1,allowDtd:!0,rawTextElements:null,implicitSelfClosingElements:null,contextSliceSize:30},t={strictDocument:!1,lowercaseNames:!0,allowDtd:!0,rawTextElements:["script","style","title","textarea"],implicitSelfClosingElements:["area","base","br","col","embed","hr","img","input","link","meta","param","source","track","wbr"],contextSliceSize:30},l={strictDocument:!1,lowercaseNames:!1,allowDtd:!0,rawTextElements:["style"],implicitSelfClosingElements:[],contextSliceSize:30};export{t as htmlConfig,l as svgConfig,e as xmlConfig};
|
package/dist/esm/html-to-md.js
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import{htmlConfig as O}from"./config.js";import{xmlToString as x,getXmlStringNodeContent as
|
|
2
|
-
`},blockquote:(n,t)=>{const r=
|
|
3
|
-
`,code:n=>{const t=
|
|
1
|
+
import{htmlConfig as O}from"./config.js";import{xmlToString as x,getXmlStringNodeContent as s}from"./xml-to-string.js";var y=Object.defineProperty,f=Object.getOwnPropertySymbols,$=Object.prototype.hasOwnProperty,p=Object.prototype.propertyIsEnumerable,h=(e,l,o)=>l in e?y(e,l,{enumerable:!0,configurable:!0,writable:!0,value:o}):e[l]=o,m=(e,l)=>{for(var o in l||(l={}))$.call(l,o)&&h(e,o,l[o]);if(f)for(var o of f(l))p.call(l,o)&&h(e,o,l[o]);return e},k=(e,l)=>{var o={};for(var c in e)$.call(e,c)&&l.indexOf(c)<0&&(o[c]=e[c]);if(e!=null&&f)for(var c of f(e))l.indexOf(c)<0&&p.call(e,c)&&(o[c]=e[c]);return o};function j(e,l={}){const o=l,{xmlOptions:c=O,transformers:b,skipNodes:g=P}=o,d=k(o,["xmlOptions","transformers","skipNodes"]);return x(e,m({xmlOptions:c,skipNodes:g,transformers:m({h1:(n,t)=>{const r=u(s(n));return r===""?"":`${i(t)}# ${r}`},h2:(n,t)=>{const r=u(s(n));return r===""?"":`${i(t)}## ${r}`},h3:(n,t)=>{const r=u(s(n));return r===""?"":`${i(t)}### ${r}`},h4:(n,t)=>{const r=u(s(n));return r===""?"":`${i(t)}#### ${r}`},h5:(n,t)=>{const r=u(s(n));return r===""?"":`${i(t)}##### ${r}`},h6:(n,t)=>{const r=u(s(n));return r===""?"":`${i(t)}###### ${r}`},p:(n,t)=>{const r=u(s(n));return r===""?"":`${i(t)}${r}`},strong:n=>{const t=u(s(n));return t===""?"":`**${t}**`},b:n=>{const t=u(s(n));return t===""?"":`**${t}**`},em:n=>{const t=u(s(n));return t===""?"":`*${t}*`},i:n=>{const t=u(s(n));return t===""?"":`*${t}*`},a:n=>{var t;const r=(t=n.attributes.find(v=>v.local==="href"))==null?void 0:t.value,a=u(s(n));return a===""?"":r!=null?`[${a}](${r})`:a},ul:(n,t)=>{const r=s(n);return r===""?"":`${i(t)}${r}`},ol:(n,t)=>{const r=s(n);return r===""?"":`${i(t)}${r}`},li:(n,t)=>{const r=u(s(n));if(r==="")return"";const a=w(t);return`${" ".repeat(Math.max(0,a-1))}- ${r}
|
|
2
|
+
`},blockquote:(n,t)=>{const r=u(s(n));return r===""?"":`${i(t)}> ${r}`},hr:(n,t)=>`${i(t)}---`,br:()=>`
|
|
3
|
+
`,code:n=>{const t=u(s(n));return t===""?"":`\`${t}\``},pre:(n,t)=>{const r=s(n);return r===""?"":`${i(t)}\`\`\`
|
|
4
4
|
${r}
|
|
5
|
-
\`\`\``}},b)},d))}function
|
|
5
|
+
\`\`\``}},b)},d))}function u(e){return e.replace(/\s+/g," ").trim()}function i(e){if(e.length===0)return"";const l=e[e.length-1];if(l==null||l.content.length===0)return"";for(let o=l.content.length-1;o>=0;o--){const c=l.content[o];if(typeof c=="object"&&c.string!=null&&c.string!==""){if(S.has(c.local))return`
|
|
6
6
|
|
|
7
|
-
`;if(N.has(
|
|
8
|
-
`}}return""}function w(e){let l=0;for(let o=e.length-1;o>=0;o--){const
|
|
7
|
+
`;if(N.has(c.local))return`
|
|
8
|
+
`}}return""}function w(e){let l=0;for(let o=e.length-1;o>=0;o--){const c=e[o];c&&(c.local==="ul"||c.local==="ol")&&l++}return l}const S=new Set(["h1","h2","h3","h4","h5","h6","p","div","blockquote","hr","pre","ul","ol","li"]),N=new Set(["strong","b","em","i","a","code","br"]),P=["script","style","meta","link","noscript","iframe","object","embed","param","base","title","head"];export{j as htmlToMarkdown};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
import{AMPERSAND as x,HASH as
|
|
1
|
+
import{AMPERSAND as x,HASH as w,LOWERCASE_X as U,ZERO as y,NINE as S,UPPERCASE_A as E,UPPERCASE_F as v,LOWERCASE_A as P,LOWERCASE_F as k,SEMICOLON as N,COLON as A,SINGLE_QUOTE as T,DOUBLE_QUOTE as O,LINE_FEED as b}from"./ascii-constants.js";import{isXmlChar as d,isXmlSpaceByte as I,isAsciiDigit as W,isXmlNameStart as l,isXmlName as m}from"./utils.js";import{XmlError as n}from"./XmlError.js";var F=Object.defineProperty,_=Object.getOwnPropertySymbols,L=Object.prototype.hasOwnProperty,R=Object.prototype.propertyIsEnumerable,p=(h,t,e)=>t in h?F(h,t,{enumerable:!0,configurable:!0,writable:!0,value:e}):h[t]=e,f=(h,t)=>{for(var e in t||(t={}))L.call(t,e)&&p(h,e,t[e]);if(_)for(var e of _(t))R.call(t,e)&&p(h,e,t[e]);return h},a=(h,t,e)=>p(h,typeof t!="symbol"?t+"":t,e);class u{constructor(t,e={}){a(this,"_text"),a(this,"_pos"),a(this,"_end"),a(this,"config");const{pos:s=0,strictDocument:o=!0,lowercaseNames:i=!1,allowDtd:r=!0,rawTextElements:c=null,implicitSelfClosingElements:C=null,contextSliceSize:g=!1}=e;this._text=t,this._pos=s,this._end=this._text.length,this.config={strictDocument:o,lowercaseNames:i,allowDtd:r,rawTextElements:c,implicitSelfClosingElements:C,contextSliceSize:g}}clone(){return new u(this._text,f({pos:this._pos},this.config))}getPos(){return this._pos}atEnd(){return this._pos>=this._end}currCodeUnit(){if(this._pos>=this._end)throw new n({type:"UnexpectedEndOfStream"});return this._text.charCodeAt(this._pos)}currCodeUnitUnchecked(){return this._text.charCodeAt(this._pos)}nextCodeUnit(){if(this._pos+1>=this._end)throw new n({type:"UnexpectedEndOfStream"});return this._text.charCodeAt(this._pos+1)}consumeCodeUnit(t){const e=this.currCodeUnit();if(e!==t)throw new n({type:"InvalidChar",expected:t,actual:e},this.genTextPos());this._pos+=1}tryConsumeCodeUnit(t){return this.currCodeUnit()===t?(this._pos+=1,!0):!1}consumeCodeUnitsWhile(t){const e=this._pos;return this.skipCodeUnitsWhile(t),this.sliceBack(e)}skipCodeUnitsWhile(t){for(;this._pos<this._end&&t(this.currCodeUnitUnchecked(),this);){if(!d(this.currCodeUnitUnchecked()))throw new n({type:"NonXmlChar",char:String.fromCodePoint(this.currCodeUnitUnchecked())},this.genTextPos());this._pos+=1}}advance(t){this._pos+=t}goTo(t){this._pos=t}goToEnd(){this._pos=this._end}startsWith(t){return this._text.startsWith(t,this._pos)}startsWithIgnoreCase(t){if(this._pos+t.length>this._end)return!1;for(let e=0;e<t.length;e++){const s=this._text.charCodeAt(this._pos+e),o=t.charCodeAt(e),i=s>=97&&s<=122?s-32:s,r=o>=97&&o<=122?o-32:o;if(i!==r)return!1}return!0}skipString(t){if(!this.startsWith(t))throw new n({type:"InvalidString",expected:t},this.genTextPos());this._pos+=t.length}sliceBack(t){return this._text.slice(t,this._pos)}rangeFrom(t){return{start:t,end:this._pos}}skipSpaces(){for(;this.startsWithSpace();)this._pos+=1}startsWithSpace(){return this._pos<this._end&&I(this.currCodeUnitUnchecked())}consumeSpaces(){if(this._pos>=this._end)throw new n({type:"UnexpectedEndOfStream"},this.genTextPos());if(!this.startsWithSpace())throw new n({type:"InvalidChar",expected:"a whitespace",actual:this.currCodeUnitUnchecked()},this.genTextPos());this.skipSpaces()}tryConsumeReference(){const t=this._pos,e=this.clone(),s=e.consumeReference();return s!=null?(this._pos+=e.getPos()-t,s):null}consumeReference(){if(!this.tryConsumeCodeUnit(x))return null;let t;if(this.tryConsumeCodeUnit(w)){let e,s;this.tryConsumeCodeUnit(U)?(e=this.consumeCodeUnitsWhile(i=>i>=y&&i<=S||i>=E&&i<=v||i>=P&&i<=k),s=16):(e=this.consumeCodeUnitsWhile(i=>W(i)),s=10);const o=parseInt(e,s);isNaN(o)||!d(o)?t=null:t={type:"Char",value:String.fromCodePoint(o)}}else{const e=this.consumeName();switch(e){case"quot":t={type:"Char",value:'"'};break;case"amp":t={type:"Char",value:"&"};break;case"apos":t={type:"Char",value:"'"};break;case"lt":t={type:"Char",value:"<"};break;case"gt":t={type:"Char",value:">"};break;default:t={type:"Entity",value:e}}}return this.tryConsumeCodeUnit(N)?t:null}consumeName(){const t=this._pos;this.skipName();const e=this.sliceBack(t);if(e.length===0)throw new n({type:"InvalidName"},this.genTextPosFrom(t));return e}skipName(){const t=this._pos;for(;this._pos<this._end;){const e=this.currCodeUnitUnchecked();if(this._pos===t){if(!l(e))throw new n({type:"InvalidName"},this.genTextPosFrom(t))}else if(!m(e))break;this._pos+=1}}consumeQName(){var t,e;const s=this._pos;let o=null;for(;this._pos<this._end;){const c=this.currCodeUnitUnchecked();if(c===A)if(o==null)o=this._pos,this._pos+=1;else throw new n({type:"InvalidName"},this.genTextPosFrom(s));else if(m(c))this._pos+=1;else break}let i,r;if(o!=null?(i=this._text.slice(s,o),r=this.sliceBack(o+1)):(i="",r=this.sliceBack(s)),i.length>0&&!l((t=i[0])==null?void 0:t.codePointAt(0)))throw new n({type:"InvalidName"},this.genTextPosFrom(s));if(r.length>0){if(!l((e=r[0])==null?void 0:e.codePointAt(0)))throw new n({type:"InvalidName"},this.genTextPosFrom(s))}else throw new n({type:"InvalidName"},this.genTextPosFrom(s));return this.config.lowercaseNames&&(i=i.replace(/[A-Z]/g,c=>c.toLowerCase()),r=r.replace(/[A-Z]/g,c=>c.toLowerCase())),[i,r]}consumeQuote(){const t=this.currCodeUnit();if(t===T||t===O)return this._pos+=1,t;throw new n({type:"InvalidChar",expected:"a quote",actual:t},this.genTextPos())}getTextAround(t,e){const s=Math.floor(e/2),o=Math.max(0,t-s),i=Math.min(this._end,t+s+e%2);return this._text.slice(o,i)}genTextPos(){return this.genTextPosFrom(this._pos)}genTextPosFrom(t){const e=Math.min(t,this._end);let s=1,o=1;for(let i=0;i<e;i++)this._text.charCodeAt(i)===b?(s++,o=1):o++;return f({row:s,col:o},this.config.contextSliceSize?{contextSlice:this.getTextAround(t,this.config.contextSliceSize)}:{})}}export{u as XmlStream};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
import{getQName as w}from"../get-q-name.js";import{LESS_THAN as
|
|
1
|
+
import{getQName as w}from"../get-q-name.js";import{LESS_THAN as u,GREATER_THAN as r,SLASH as f,EQUALS as W,SINGLE_QUOTE as y,DOUBLE_QUOTE as P,HYPHEN as X,QUESTION_MARK as x,OPEN_SQUARE_BRACKET as H,PERCENT as Y,EXCLAMATION_MARK as K,UPPERCASE_S as q,UPPERCASE_P as z,CLOSE_SQUARE_BRACKET as G}from"./ascii-constants.js";import{isXmlSpaceByte as $}from"./utils.js";import{XmlError as a}from"./XmlError.js";import{XmlStream as j}from"./XmlStream.js";const J="\uFEFF",v="<?xml ",I="?>",Z="<!ENTITY",V="<!ELEMENT",tt="<!ATTLIST",nt="<!NOTATION",g="<!DOCTYPE",A="<?",N="?>",h="<!--",D="-->",st="<![CDATA[",C="]]>",et="NDATA",F="PUBLIC",ot="SYSTEM",it="version",ct="encoding",at="standalone";function rt(t,n,s={}){Q(new j(t,s),n)}function Q(t,n){if(t.startsWith(J)&&t.advance(1),t.startsWith(v)&&pt(t),d(t,n),t.config.strictDocument){if(t.skipSpaces(),t.startsWithIgnoreCase(g)&&(U(t,n),d(t,n)),t.skipSpaces(),!t.atEnd()&&t.currCodeUnit()===u&&k(t,n),d(t,n),!t.atEnd())throw new a({type:"UnknownToken",message:"Not at end"},t.genTextPos())}else for(;!t.atEnd();)t.currCodeUnit()===u?k(t,n):O(t,n),d(t,n)}function d(t,n){for(;!t.atEnd();)if(t.skipSpaces(),t.startsWith(h))E(t,n);else if(!t.config.strictDocument&&t.startsWithIgnoreCase(g))U(t,n);else if(t.startsWith(A))S(t,n);else break}function pt(t){function n(s){if(s.startsWithSpace())s.skipSpaces();else if(!s.startsWith(I)&&!s.atEnd())throw new a({type:"InvalidChar",expected:"a whitespace",actual:s.currCodeUnitUnchecked()},s.genTextPos())}if(t.advance(5),n(t),!t.startsWith(it))throw new a({type:"InvalidString",expected:"version"},t.genTextPos());m(t),n(t),t.startsWith(ct)&&(m(t),n(t)),t.startsWith(at)&&m(t),t.skipSpaces(),t.skipString(I)}function E(t,n){const s=t.getPos();t.advance(4);const e=t.consumeCodeUnitsWhile((o,i)=>!(o===X&&i.startsWith(D)));if(t.skipString(D),e.includes("--")||e.endsWith("-"))throw new a({type:"InvalidComment"},t.genTextPosFrom(s));n({type:"Comment",text:e,range:t.rangeFrom(s)},t)}function S(t,n){if(t.startsWith(v))throw new a({type:"UnexpectedDeclaration"},t.genTextPos());const s=t.getPos();t.advance(2);const e=t.consumeName();t.skipSpaces();const o=t.consumeCodeUnitsWhile((i,c)=>!(i===x&&c.startsWith(N)));t.skipString(N),n({type:"ProcessingInstruction",target:e,content:o.length===0?void 0:o,range:t.rangeFrom(s)},t)}function U(t,n){if(!t.config.allowDtd)throw new a({type:"DtdDetected"});const s=t.getPos();if(lt(t),t.skipSpaces(),t.currCodeUnit()===r){t.advance(1);return}for(t.advance(1);!t.atEnd();)if(t.skipSpaces(),t.startsWith(Z))ut(t,n);else if(t.startsWith(h))E(t,n);else if(t.startsWith(A))S(t,n);else if(t.startsWith("]"))if(t.advance(1),t.skipSpaces(),t.currCodeUnit()===r){t.advance(1);break}else throw new a({type:"InvalidChar",expected:"'>'",actual:t.currCodeUnitUnchecked()},t.genTextPos());else if(t.startsWith(V)||t.startsWith(tt)||t.startsWith(nt))try{mt(t)}catch(e){throw new a({type:"UnknownToken",message:"Failed to consume declaration"},t.genTextPosFrom(s))}else throw new a({type:"UnknownToken",message:"Failed to parse doctype"},t.genTextPos())}function lt(t){t.advance(9),t.consumeSpaces(),t.skipName(),t.skipSpaces(),_(t),t.skipSpaces();const n=t.currCodeUnit();if(n!==H&&n!==r)throw new a({type:"InvalidChar",expected:"'[' or '>'",actual:n},t.genTextPos())}function _(t){if(t.startsWith(ot)||t.startsWith(F)){const n=t.getPos();t.advance(6);const s=t.sliceBack(n);t.consumeSpaces();const e=t.consumeQuote();if(t.consumeCodeUnitsWhile(o=>o!==e),t.consumeCodeUnit(e),s===F){t.consumeSpaces();const o=t.consumeQuote();t.consumeCodeUnitsWhile(i=>i!==o),t.consumeCodeUnit(o)}return!0}return!1}function ut(t,n){t.advance(8),t.consumeSpaces();const s=!t.tryConsumeCodeUnit(Y);s||t.consumeSpaces();const e=t.consumeName();t.consumeSpaces();const o=dt(t,s);o!==null&&n({type:"EntityDeclaration",name:e,definition:o},t),t.skipSpaces(),t.consumeCodeUnit(r)}function dt(t,n){const s=t.currCodeUnit();if(s===P||s===y){const e=t.consumeQuote(),o=t.getPos();t.skipCodeUnitsWhile(c=>c!==e);const i=t.sliceBack(o);return t.consumeCodeUnit(e),i}else if(s===q||s===z){if(_(t))return n&&(t.skipSpaces(),t.startsWith(et)&&(t.advance(5),t.consumeSpaces(),t.skipName())),null;throw new a({type:"InvalidExternalID"},t.genTextPos())}else throw new a({type:"InvalidChar",expected:"a quote, SYSTEM or PUBLIC",actual:s},t.genTextPos())}function mt(t){t.skipCodeUnitsWhile(n=>n!==r),t.consumeCodeUnit(r)}function k(t,n){const s=t.getPos();t.advance(1);const[e,o]=t.consumeQName();n({type:"ElementStart",prefix:e,local:o,start:s},t);let i=!1;for(;!t.atEnd();){const c=t.startsWithSpace();t.skipSpaces();const p=t.getPos(),T=t.currCodeUnit();if(T===f){t.advance(1),t.consumeCodeUnit(r);const l=t.rangeFrom(p);n({type:"ElementEnd",end:{type:"Empty"},range:l},t);break}else if(T===r){t.advance(1);const l=t.rangeFrom(p);if(t.config.implicitSelfClosingElements!=null&&t.config.implicitSelfClosingElements.includes(w(o,e))){n({type:"ElementEnd",end:{type:"Empty"},range:l},t);break}else{n({type:"ElementEnd",end:{type:"Open"},range:l},t),i=!0;break}}else{if(!c)throw new a({type:"InvalidChar",expected:"a whitespace",actual:t.currCodeUnitUnchecked()},t.genTextPos());const[l,b,B]=m(t),M=t.getPos();n({type:"Attribute",range:{start:p,end:M},prefix:l,local:b,value:B},t)}}if(i){if(t.config.rawTextElements!=null){const c=w(o,e);if(t.config.rawTextElements.includes(c)){L(t,n,`</${c}>`);return}}L(t,n)}}function m(t){const[n,s]=t.consumeQName();let e;const o=t.getPos();if(t.skipSpaces(),t.tryConsumeCodeUnit(W)){t.skipSpaces();const i=t.currCodeUnit();if(i===y||i===P){const c=t.consumeQuote();e=t.consumeCodeUnitsWhile(p=>p!==c&&p!==u),t.consumeCodeUnit(c)}else if(!t.config.strictDocument)e=t.consumeCodeUnitsWhile(c=>!$(c)&&c!==r&&c!==f&&c!==u);else throw new a({type:"InvalidChar",expected:"a quote",actual:i},t.genTextPos())}else if(!t.config.strictDocument)t.goTo(o),e="true";else throw new a({type:"InvalidChar",expected:W,actual:t.currCodeUnit()},t.genTextPos());return[n,s,e]}function L(t,n,s=u){for(;!t.atEnd();)if(R(t,s)){const e=t.nextCodeUnit();if(e===K)if(t.startsWith(h))E(t,n);else if(t.startsWith(st))ft(t,n);else if(!t.config.strictDocument&&t.startsWithIgnoreCase(g))U(t,n);else throw new a({type:"UnknownToken",message:"Failed to parse content"},t.genTextPos());else if(e===x)S(t,n);else if(e===f){gt(t,n);break}else k(t,n)}else O(t,n,s)}function ft(t,n){const s=t.getPos();t.advance(9);const e=t.consumeCodeUnitsWhile((i,c)=>!(i===G&&c.startsWith(C)));t.skipString(C);const o=t.rangeFrom(s);n({type:"Cdata",text:e,range:o},t)}function gt(t,n){const s=t.getPos();t.advance(2);const[e,o]=t.consumeQName();t.skipSpaces(),t.consumeCodeUnit(r);const i=t.rangeFrom(s);n({type:"ElementEnd",end:{type:"Close",prefix:e,local:o},range:i},t)}function O(t,n,s=u){const e=t.getPos(),o=typeof s=="number"?t.consumeCodeUnitsWhile(i=>i!==s):t.consumeCodeUnitsWhile((i,c)=>!R(c,s));if(t.config.strictDocument&&o.includes(C))throw new a({type:"InvalidCharacterData"},t.genTextPos());n({type:"Text",text:o,range:t.rangeFrom(e)},t)}function R(t,n){return typeof n=="number"?t.currCodeUnit()===n:t.config.lowercaseNames?t.startsWithIgnoreCase(n):t.startsWith(n)}export{rt as tokenize,Q as tokenizeXmlStream};
|
package/dist/types/config.d.ts
CHANGED
|
@@ -6,6 +6,7 @@ export declare const xmlConfig: TXmlStreamOptions;
|
|
|
6
6
|
/**
|
|
7
7
|
* HTML configuration with HTML-specific settings.
|
|
8
8
|
* - Non-strict document structure
|
|
9
|
+
* - ASCII lowercase element and attribute names
|
|
9
10
|
* - Raw text elements like script and style
|
|
10
11
|
* - Implicit self-closing elements (void elements in HTML)
|
|
11
12
|
*/
|
|
@@ -102,10 +102,10 @@ export declare class XmlStream {
|
|
|
102
102
|
*/
|
|
103
103
|
startsWith(text: string): boolean;
|
|
104
104
|
/**
|
|
105
|
-
* Checks
|
|
105
|
+
* Checks for text at the current position, ignoring ASCII letter case on both sides.
|
|
106
|
+
* Compares non-ASCII characters exactly and leaves the stream position unchanged.
|
|
106
107
|
*
|
|
107
|
-
* @param text - The text to check for
|
|
108
|
-
* @returns True if the stream starts with the text (case-insensitive), false otherwise.
|
|
108
|
+
* @param text - The text to check for, in any letter case.
|
|
109
109
|
*/
|
|
110
110
|
startsWithIgnoreCase(text: string): boolean;
|
|
111
111
|
/**
|
|
@@ -227,6 +227,13 @@ export interface TXmlStreamConfig {
|
|
|
227
227
|
* @default true
|
|
228
228
|
*/
|
|
229
229
|
strictDocument: boolean;
|
|
230
|
+
/**
|
|
231
|
+
* Convert ASCII A-Z to a-z in opening-tag, closing-tag, and attribute names, including prefixes.
|
|
232
|
+
* Also match configured raw text closing tags ignoring ASCII case.
|
|
233
|
+
* Preserve non-ASCII characters, text content, attribute values, and source positions.
|
|
234
|
+
* @default false
|
|
235
|
+
*/
|
|
236
|
+
lowercaseNames: boolean;
|
|
230
237
|
/**
|
|
231
238
|
* List of element names that should be treated as raw text elements.
|
|
232
239
|
* Their content will be parsed as text rather than XML.
|
package/package.json
CHANGED
|
@@ -1,9 +1,23 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "xml-tokenizer",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.47",
|
|
4
4
|
"private": false,
|
|
5
|
-
"description": "
|
|
6
|
-
"keywords": [
|
|
5
|
+
"description": "Streaming XML, HTML, and SVG tokenizer with typed tokens, selectors, and object helpers",
|
|
6
|
+
"keywords": [
|
|
7
|
+
"xml",
|
|
8
|
+
"html",
|
|
9
|
+
"svg",
|
|
10
|
+
"tokenizer",
|
|
11
|
+
"parser",
|
|
12
|
+
"xml-parser",
|
|
13
|
+
"html-parser",
|
|
14
|
+
"streaming-parser",
|
|
15
|
+
"sax",
|
|
16
|
+
"typescript",
|
|
17
|
+
"selector",
|
|
18
|
+
"object-parser",
|
|
19
|
+
"markup"
|
|
20
|
+
],
|
|
7
21
|
"homepage": "https://builder.group/?utm_source=package-json",
|
|
8
22
|
"bugs": {
|
|
9
23
|
"url": "https://github.com/builder-group/community/issues"
|
|
@@ -25,14 +39,14 @@
|
|
|
25
39
|
"devDependencies": {
|
|
26
40
|
"@types/sax": "^1.2.7",
|
|
27
41
|
"@types/xml2js": "^0.4.14",
|
|
28
|
-
"camaro": "^6.2
|
|
29
|
-
"fast-xml-parser": "^5.
|
|
30
|
-
"sax": "^1.6.
|
|
31
|
-
"saxen": "^11.
|
|
32
|
-
"txml": "^
|
|
42
|
+
"camaro": "^6.7.2",
|
|
43
|
+
"fast-xml-parser": "^5.11.1",
|
|
44
|
+
"sax": "^1.6.1",
|
|
45
|
+
"saxen": "^11.1.1",
|
|
46
|
+
"txml": "^6.0.3",
|
|
33
47
|
"xml2js": "^0.6.2",
|
|
34
|
-
"@blgc/config": "0.
|
|
35
|
-
"rollup-presets": "0.0.
|
|
48
|
+
"@blgc/config": "0.1.2",
|
|
49
|
+
"rollup-presets": "0.0.30"
|
|
36
50
|
},
|
|
37
51
|
"size-limit": [
|
|
38
52
|
{
|
|
@@ -45,8 +59,8 @@
|
|
|
45
59
|
"build:prod": "export NODE_ENV=production && pnpm build",
|
|
46
60
|
"clean": "shx rm -rf dist && shx rm -rf .turbo && shx rm -rf node_modules",
|
|
47
61
|
"install:clean": "pnpm run clean && pnpm install",
|
|
48
|
-
"lint": "eslint .
|
|
49
|
-
"publish:patch": "pnpm build:prod && pnpm version patch && pnpm publish --no-git-checks --access=public",
|
|
62
|
+
"lint": "eslint .",
|
|
63
|
+
"publish:patch": "pnpm build:prod && pnpm version patch --no-git-tag-version && pnpm publish --no-git-checks --access=public",
|
|
50
64
|
"size": "size-limit --why",
|
|
51
65
|
"start:dev": "tsc -w",
|
|
52
66
|
"test": "vitest run",
|