xml-toolkit 0.0.1

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2021 do-
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,4 @@
+# node-xml-toolkit
+Collection of classes for dealing with XML
+
+This is a project template. The actual code is yet to be written.

package/index.js

@@ -0,0 +1,5 @@
+const XMLLexer        = require ('./lib/XMLLexer')
+const SAXEventEmitter = require ('./lib/SAXEventEmitter')
+const SAXEvent        = require ('./lib/SAXEvent')
+
+module.exports = {SAXEventEmitter, XMLLexer, SAXEvent}

package/lib/SAXEvent.js

@@ -0,0 +1,166 @@
+const assert     = require ('assert')
+
+const CH_LT          = '<'.charCodeAt (0)
+const CH_EXCLAMATION = '!'.charCodeAt (0)
+const CH_QUESTION    = '?'.charCodeAt (0)
+const CH_SLASH       = '/'.charCodeAt (0)
+const CH_MINUS       = '-'.charCodeAt (0)
+
+const STR_XML        = 'XML'
+const STR_CDATA      = '[CDATA['
+const STR_DOCTYPE    = 'DOCTYPE'
+
+const START_DOCUMENT         = 'StartDocument'
+const PROCESSING_INSTRUCTION = 'ProcessingInstruction'
+const COMMENT                = 'Comment'
+const DTD                    = 'DTD'
+const START_ELEMENT          = 'StartElement'
+const CHARACTERS             = 'Characters'
+const END_ELEMENT            = 'EndElement'
+const END_DOCUMENT           = 'EndDocument'
+const CDATA                  = 'CDATA'
+
+const SAXEvent = class {
+
+	constructor (src) {
+
+		this.src        = src
+		
+		this._afterName = 0
+
+	}
+
+	get type () {
+	
+		const {src} = this
+
+		switch (src.charCodeAt (0)) {
+
+			case CH_LT:
+
+				switch (src.charCodeAt (1)) {
+
+					case CH_SLASH: 
+
+						return END_ELEMENT
+
+					case CH_QUESTION: 
+
+						switch (src.slice (2, 5).toLowerCase ()) {
+
+							case STR_XML: return START_DOCUMENT
+
+							default: return PROCESSING_INSTRUCTION
+
+						}
+
+					case CH_EXCLAMATION:
+
+						if (src.charCodeAt (2) === CH_MINUS) return COMMENT
+
+						switch (src.slice (2, 9)) {					
+
+							case STR_CDATA: return CDATA
+
+							case STR_DOCTYPE: return DTD
+
+							default: return null
+
+						}
+
+					default: return START_ELEMENT
+
+				}
+
+				break
+
+			default:
+
+				return CHARACTERS
+
+		}
+
+	}
+	
+	get text () {
+
+		const {src} = this
+
+		switch (src.charCodeAt (0)) {
+
+			case CH_LT: return src.slice (9, -3)
+			
+			default: return src
+			
+		}
+
+	}
+
+	get attributes () {
+	
+		const m = new Map ()
+		
+		this.writeAttributesToMap (m)
+		
+		return m
+	
+	}
+	
+	writeAttributesToMap (m) {
+	
+		const {src} = this
+
+		let start = this._afterName; while (true) {
+
+			const eq = src.indexOf ('=', start); if (eq === -1) break
+
+			const  q = src.indexOf (src.charAt (eq + 1), eq + 2); if (q === -1) break
+
+			let k = src.slice (start, eq)
+			
+			if (start === 0) {
+			
+				let p = k.length - 2; while (p !== 0 && k.charCodeAt (p) <= 32) p --
+
+				k = k.slice (this._afterName = p)
+
+			}
+			else {
+			
+				k = k.trim ()
+			
+			}			
+
+			m.set (k, src.slice (eq + 2, q))
+
+			start = q + 1
+		
+		}
+		
+		return m
+	
+	}
+	
+	get isSelfEnclosed () {
+
+		const {src} = this
+
+		return src.charCodeAt (src.length - 2) === CH_SLASH
+	
+	}
+
+}
+
+SAXEvent.TYPES = {
+	START_DOCUMENT,
+	PROCESSING_INSTRUCTION,
+	COMMENT,
+	DTD,
+	START_ELEMENT,
+	CHARACTERS,         
+	END_ELEMENT,
+	END_DOCUMENT,
+	CDATA,
+}
+
+module.exports = SAXEvent

package/lib/SAXEventEmitter.js

@@ -0,0 +1,42 @@
+const assert     = require ('assert')
+const {Writable} = require ('stream')
+const SAXEvent   = require ('./SAXEvent.js')
+
+const SAXEventEmitter = class extends Writable {
+
+	constructor (options = {}) {
+
+		options.decodeStrings = false
+
+		super (options)
+		
+		this.on ('finish', () => this.emit (SAXEvent.TYPES.END_DOCUMENT))
+
+	}
+
+	_write (chunk, encoding, callback) {
+
+		if (chunk.length !== 0) {
+		
+			let e = new SAXEvent (chunk), {type} = e
+			
+			if (type === SAXEvent.TYPES.CDATA) {
+				e = new SAXEvent (e.text)
+				type = SAXEvent.TYPES.CHARACTERS
+			}
+
+			this.emit (type, e)
+		
+			if (type === SAXEvent.TYPES.TYPE_START_ELEMENT && e.isSelfEnclosing ()) {
+				this.emit (SAXEvent.TYPES.TYPE_END_ELEMENT, e)
+			}
+		
+		}
+
+		callback ()
+
+	}
+
+}
+
+module.exports = SAXEventEmitter

package/lib/XMLLexer.js

@@ -0,0 +1,292 @@
+const assert      = require ('assert')
+const {Transform} = require ('stream')
+const {StringDecoder} = require('string_decoder')
+
+const ST_TEXT  = 0
+const ST_LT	   = 1
+const ST_LT_X  = 2
+const ST_TAG   = 3
+const ST_TAG_X = 4
+
+const CH_EXCLAMATION = '!'.charCodeAt (0)
+const CH_QUESTION    = '?'.charCodeAt (0)
+const CH_MINUS       = '-'.charCodeAt (0)
+const CH_SQUARE      = '['.charCodeAt (0)
+const CH_GT          = '>'.charCodeAt (0)
+
+const CL_DEAFULT = Buffer.from ('')
+const CL_PI      = Buffer.from ('?',  'ascii')
+const CL_COMMENT = Buffer.from ('--', 'ascii')
+const CL_SQ_1    = Buffer.from (']',  'ascii')
+const CL_SQ_2    = Buffer.from (']]', 'ascii')
+
+const S_ENCODING = Symbol ('encoding')
+
+const XMLLexer = class extends Transform {
+
+	constructor (options = {}) {
+
+		options.readableObjectMode = true
+		options.decodeStrings = false
+		
+		let encoding = options.encoding; delete options.encoding
+		if (encoding == null) encoding = 'utf8'
+		
+		let maxLength = options.maxLength; delete options.maxLength
+		if (maxLength == null) maxLength = 1 << 20
+		
+		assert (Number.isInteger (maxLength), 'maxLength must be integer, not ' + maxLength)
+		assert (maxLength > 0, 'maxLength must be positive, not ' + maxLength)
+		
+		if (!('stripSpace' in options)) options.stripSpace = false
+		assert (options.stripSpace === true || options.stripSpace === false, 'options.stripSpace must be boolean, not ' + typeof options.stripSpace)
+
+		super (options)
+		
+		this.decoder      = new StringDecoder ('utf8')
+
+		this.stripSpace   = options.stripSpace
+		this.body		  = ''
+		this.start        = 0
+		this.setState (ST_TEXT)
+		this.beforeBody   = 0n
+		
+		this [S_ENCODING] = encoding
+		this.maxLength    = maxLength
+
+	}
+
+	isClosing (pos) {
+
+		const {awaited} = this, {length} = awaited; if (length === 0) return true
+
+		let from = pos - length; if (from < this.start) return false
+
+		if (this.body.charCodeAt (from) !== awaited [0]) return false
+
+		if (length === 2 && this.body.charCodeAt (from + 1) !== awaited [1]) return false
+		
+		return true
+	
+	}
+	
+	setState (state, awaited = null) {
+
+		switch (state) {
+
+			case ST_TAG:
+			case ST_TAG_X:
+				assert (awaited != null, 'ST_TAG[_X]: awaited is mandatory')
+				break
+
+			default:
+				assert (awaited === null, 'not ST_TAG[_X]: awaited must be null')
+				this.position = 0
+				break
+
+		}
+
+		this.state      = state
+		this.awaited	= awaited
+
+	}
+
+	parse () {	
+	
+		outer: while (true) {
+		
+			const {start, body} = this
+
+			switch (this.state) {
+			
+				case ST_TEXT:
+				
+					{
+
+						const pos = body.indexOf ('<', start); switch (pos) {
+
+							case -1:
+								return
+
+							case  0:
+								this.setState (ST_LT)
+								break
+
+							default:
+								this.publishTo (pos - 1)
+								this.setState (ST_LT)
+
+						}
+
+					}
+									
+					break
+					
+				case ST_LT: 
+				
+					{
+
+						const pos = start + 1; if (pos >= body.length) return null
+
+						switch (body.charCodeAt (pos)) {
+
+							case CH_EXCLAMATION:
+								this.setState (ST_LT_X)
+								break
+
+							case CH_QUESTION:
+								this.setState (ST_TAG, CL_PI)
+								break
+
+							default:
+								this.setState (ST_TAG, CL_DEAFULT)
+								break
+						}
+
+					}
+				
+					
+					break
+					
+				case ST_LT_X:
+				
+					{
+
+						const pos = start + 2; if (pos >= body.length) return null
+						
+						switch (body.charCodeAt (pos)) {
+						
+							case CH_MINUS:
+								this.setState (ST_TAG, CL_COMMENT)
+								break
+								
+							default:
+								this.setState (ST_TAG_X, CL_DEAFULT)
+								this.position = pos
+								break
+						
+						}
+
+					}
+				
+					break
+					
+				case ST_TAG:
+				
+					let pos = start; while (true) {
+
+						pos = body.indexOf ('>', pos + 1); if (pos === -1) return
+						
+						if (!this.isClosing (pos)) continue
+
+						this.publishTo (pos)
+
+						this.setState (ST_TEXT)
+
+						continue outer							
+
+					}
+				
+				case ST_TAG_X:
+				
+					for (let pos = this.position + 1; pos < body.length; pos ++) {
+					
+						switch (body.charCodeAt (pos)) {
+						
+							case CH_SQUARE:
+								
+								switch (this.awaited) {
+								
+									case CL_DEAFULT:
+										this.awaited = CL_SQ_1
+										break
+										
+									case CL_SQ_1:
+										this.awaited = CL_SQ_2
+										break
+
+								}
+								
+								this.position = pos
+								break
+
+							case CH_GT:
+								if (!this.isClosing (pos)) break
+								this.publishTo (pos)
+								this.setState (ST_TEXT)
+								continue outer
+							
+						}
+
+					}
+					
+					return
+
+				default:
+					throw new Error ('Invalid state: ' + this.state)
+					
+			}
+
+		}	
+		
+	}
+
+	publishTo (pos) {
+	
+		let lexeme = this.body.slice (this.start, ++ pos)
+
+		this.start = pos
+		
+		if (this.stripSpace) lexeme = lexeme.trim ()
+
+		if (lexeme.length !== 0) this.push (lexeme)
+
+	}
+
+	_flush (callback) {
+	
+		this.publishTo (this.body.length - 1)
+		
+		callback ()
+
+	}
+	
+	getPosition () {
+	
+		let {beforeBody, start} = this
+		
+		return start === 0 ? beforeBody : beforeBody + BigInt (start)
+	
+	}
+	
+	checkMaxLength () {
+	
+		const {body, maxLength} = this, size = body.length - this.start; 
+		
+		if (size <= maxLength) return
+
+		const s = body.slice (0, size - 1), dump = JSON.stringify ([s])
+
+		this.destroy (new Error (`The fragment ${dump} at position ${this.getPosition ()} exceeds maxLength=${maxLength}`))
+	
+	}
+	
+	_transform (chunk, encoding, callback) {
+	
+		if (Buffer.isBuffer (chunk)) chunk = this.decoder.write (chunk)
+
+		this.body = this.body.slice (this.start) + chunk
+		this.start = 0
+		
+		this.parse ()
+		
+		if (this.start > 0) this.beforeBody += BigInt (this.start)		
+		this.checkMaxLength ()
+		
+		callback ()
+
+	}
+	
+}
+
+module.exports = XMLLexer

package/package.json

@@ -0,0 +1,22 @@
+{
+  "name": "xml-toolkit",
+  "version": "0.0.1",
+  "description": "Collection of classes for dealing with XML",
+  "main": "index.js",
+  "scripts": {
+    "test": "node test/test.js" 
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/do-/node-xml-toolkit.git"
+  },
+  "keywords": [
+    "xml sax stax"
+  ],
+  "author": "Dmitry Ovsyanko",
+  "license": "MIT",
+  "bugs": {
+    "url": "https://github.com/do-/node-xml-toolkit/issues"
+  },
+  "homepage": "https://github.com/do-/node-xml-toolkit#readme"
+}

package/test/E05a.xml

@@ -0,0 +1,5 @@
+<!-- CDATA sections may occur in Mixed content. -->
+<!DOCTYPE foo [
+<!ELEMENT foo (#PCDATA|foo)*>
+]>
+<foo>a <![CDATA[cdata section]]> in mixed content</foo>

package/test/not-sa01.xml

@@ -0,0 +1,10 @@
+<?xml version='1.0' standalone='no'?>
+
+<!DOCTYPE root SYSTEM "sa.dtd">
+
+<root>
+    <child>
+    The whitespace before and after this element keeps
+    this from being standalone.
+    </child>
+</root>

package/test/not-sa02.xml

@@ -0,0 +1,30 @@
+<?xml version='1.0' standalone='no'?>
+
+<!DOCTYPE attributes SYSTEM "../valid/sa.dtd" [
+    <!ENTITY internal " number99">
+]>
+
+    <!-- sync with ../invalid/not-sa02.xml -->
+
+    <!--
+	lots of normalized/defaulted attributes
+	keep this from being standalone
+
+	XXX not the best basis for negative tests!!
+    -->
+
+<attributes
+    notation =	" nonce "
+    nmtoken =	" this-gets-normalized "
+    nmtokens =	" this 	
+ also	 gets normalized "
+    id =	"	&internal; "
+    idref =	" &internal;
+    "
+    idrefs =	" &internal;  &internal;    &internal;"
+    entity =	" unparsed-1 "
+    entities =	"unparsed-1
+    unparsed-2		
+"
+    cdata =	"nothing happens to this one!"
+    />

package/test/param_types.xml

@@ -0,0 +1 @@
+<?xml version="1.0" encoding="utf-8"?><PARAMTYPES><PARAMTYPE ID="1" NAME="ИФНС ФЛ" DESC="ИФНС ФЛ" CODE="IFNSFL" ISACTIVE="true" UPDATEDATE="2018-06-15" STARTDATE="2011-11-01" ENDDATE="2079-06-06" /><PARAMTYPE ID="2" NAME="ИФНС ЮЛ" DESC="ИФНС ЮЛ" CODE="IFNSUL" ISACTIVE="true" UPDATEDATE="2018-06-15" STARTDATE="2011-11-01" ENDDATE="2079-06-06" /><PARAMTYPE ID="3" NAME="ИНН ФЛ ТЕР УЧ" DESC="Территориальный участок ИФНС ЮЛ" CODE="territorialifnsflcode" ISACTIVE="true" UPDATEDATE="2018-06-15" STARTDATE="2011-11-01" ENDDATE="2079-06-06" /><PARAMTYPE ID="4" NAME="ИФНС ЮЛ ТЕР УЧ" DESC="Территориальный участок ИФНС ФЛ" CODE="territorialifnsulcode" ISACTIVE="true" UPDATEDATE="2018-06-15" STARTDATE="2011-11-01" ENDDATE="2079-06-06" /><PARAMTYPE ID="5" NAME="Почтовый индекс" DESC="Информация о почтовом индексе" CODE="PostIndex" ISACTIVE="true" UPDATEDATE="2018-06-15" STARTDATE="2011-11-01" ENDDATE="2079-06-06" /><PARAMTYPE ID="6" NAME="ОКАТО" DESC="ОКАТО" CODE="OKATO" ISACTIVE="true" UPDATEDATE="2018-06-19" STARTDATE="2011-11-01" ENDDATE="2079-06-06" /><PARAMTYPE ID="7" NAME="OKTMO" DESC="OKTMO" CODE="OKTMO" ISACTIVE="true" UPDATEDATE="2018-06-19" STARTDATE="2011-11-01" ENDDATE="2079-06-06" /><PARAMTYPE ID="8" NAME="Кадастровый номер" DESC="Кадастровый номер" CODE="CadastrNum" ISACTIVE="true" UPDATEDATE="2018-06-19" STARTDATE="2011-11-01" ENDDATE="2079-06-06" /><PARAMTYPE ID="10" NAME="Код КЛАДР" DESC="Код адресного объекта одной строкой с признаком актуальности" CODE="CODE" ISACTIVE="true" UPDATEDATE="2018-06-21" STARTDATE="2011-11-01" ENDDATE="2079-06-06" /><PARAMTYPE ID="11" NAME="Код КЛАДР без признака актуальности" DESC="Код адресного объекта одной строкой без признака актуальности" CODE="PLAINCODE" ISACTIVE="true" UPDATEDATE="2018-06-21" STARTDATE="2011-11-01" ENDDATE="2079-06-06" /><PARAMTYPE ID="12" NAME="Код региона" DESC="Код региона" CODE="REGIONCODE" ISACTIVE="true" UPDATEDATE="2018-06-21" STARTDATE="2011-11-01" ENDDATE="2079-06-06" /><PARAMTYPE ID="13" NAME="Реестровый номер" DESC="Реестровый номер адресного объекта" CODE="ReestrNum" ISACTIVE="true" UPDATEDATE="2018-11-12" STARTDATE="2011-11-01" ENDDATE="2079-06-06" /><PARAMTYPE ID="14" NAME="Признак присвоения адреса" DESC="Признак в каком делении присвоен адрес, муниципальном/административном" CODE="DivisionType" ISACTIVE="true" UPDATEDATE="2018-12-14" STARTDATE="2011-11-01" ENDDATE="2079-06-06" /><PARAMTYPE ID="15" NAME="Порядковый номер" DESC="Порядковый номер обьекта в рамках родителя" CODE="Counter" ISACTIVE="true" UPDATEDATE="2018-12-14" STARTDATE="2011-11-01" ENDDATE="2079-06-06" /><PARAMTYPE ID="16" NAME="Официальное наименование" DESC="Официальное наименование объекта(Региона)" CODE="Official" ISACTIVE="true" UPDATEDATE="2019-12-04" STARTDATE="2019-12-04" ENDDATE="2079-06-06" /></PARAMTYPES>

package/test/test.js

@@ -0,0 +1,105 @@
+const fs = require ('fs')
+const assert = require ('assert')
+const {SAXEventEmitter, SAXEvent, XMLLexer} = require ('../')
+
+async function test_001_lexer_sync (fn) {
+
+	const xml = fs.readFileSync (
+		'test/' + fn
+//		, 'utf-8'
+	)
+	
+console.log (xml)
+
+	const lexer = new XMLLexer ({
+//		maxLength: 40,
+//		encoding: 'ascii',
+	})
+
+	lexer.on ('data', data => console.log (new SAXEvent (data).attributes))
+	
+//	for (let c of xml) lexer.write (c); lexer.end ()
+//	for (let c of xml) lexer.write (Buffer.from ([c])); lexer.end ()
+
+	lexer.end (xml)
+
+}
+
+async function test_002_lexer_stream (fn) {
+
+	const is = fs.createReadStream ('test/' + fn, {
+//		encoding: 'utf8',
+	})
+	
+	const lexer = new XMLLexer ({
+//		maxLength: 40,
+//		encoding: 'ascii',
+	})
+
+	is.pipe (lexer)
+	
+	for await (const i of lexer) 
+//	if (/^<PARAMTYPE /.test (i)) 
+	{
+		console.log ({i})
+	
+	}
+
+}
+
+
+async function test_003_emitter_sync (fn) {
+
+	const xml = fs.readFileSync (
+		'test/' + fn
+//		, 'utf-8'
+	)
+	
+console.log (xml)
+
+	const lex = new XMLLexer ({
+//		maxLength: 40,
+//		encoding: 'ascii',
+		stripSpace: true,
+	})
+
+	const sax = new SAXEventEmitter ({})
+
+	lex.pipe (sax)
+
+	for (let event of [
+		'StartDocument',
+		'ProcessingInstruction',
+		'Comment',
+		'DTD',
+		'StartElement',
+		'Characters',
+		'EndElement',
+		'EndDocument',
+	]) sax.on (event, data => console.log ([event, data]))
+
+//	lexer.on ('data', data => console.log ({data}))
+	
+//	for (let c of xml) lexer.write (c); lexer.end ()
+//	for (let c of xml) lexer.write (Buffer.from ([c])); lexer.end ()
+
+	lex.end (xml)
+
+}
+
+async function main () {
+
+//	await test_001_lexer_sync ('E05a.xml')
+//	await test_001_lexer_sync ('not-sa01.xml')
+//	await test_001_lexer_sync ('not-sa02.xml')
+	await test_001_lexer_sync ('param_types.xml')
+//	await test_002_lexer_stream ('E05a.xml')
+//	await test_002_lexer_stream ('param_types.xml')
+//	await test_002_lexer_stream ('not-sa02.xml')
+//	await test_003_emitter_sync ('E05a.xml')
+//	await test_003_emitter_sync ('param_types.xml')
+//	await test_003_emitter_sync ('not-sa01.xml')
+
+}
+
+main ()