xml-stream-editor 0.1.0 → 0.2.0

package/CHANGELOG.md

@@ -0,0 +1,24 @@
+CHANGELOG
+===
+
+0.2.0
+---
+
+Significantly improve performance of selector matching.
+
+Add validation checks for created or modified XML element attribute names.
+
+Add config option, currently just with 1. the ability to disable validation
+of outgoing XML, and 2. configuring the "saxes" parser.
+
+Hopefully more helpful, additional text in README.md.
+
+0.1.1
+---
+
+Whoops, include the finished README.md and this file (CHANGELOG.md).
+
+0.1.0
+---
+
+Initial release

package/README.md

@@ -1,5 +1,4 @@
-xml-stream-editor
-===
+# xml-stream-editor
 
 Library to edit xml files in a streaming manner. Inspired by
 [xml-stream](https://www.npmjs.com/package/xml-stream), but 1. allows using
@@ -11,23 +10,86 @@ allows you to modify XML without needing to buffer the XML files in memory.
 For small to mid-sized XML files buffering is fine. But when editing very large
 files (e.g., multi-Gb files) buffering can be a problem or an absolute blocker.
 
-Usage
----
+## Usage
 
 `xml-stream-editor` is designed to be used with node's stream systems
 by subclassing [`stream.Transform`](https://nodejs.org/api/stream.html#class-streamtransform),
 so it can be used with the [streams promises API](https://nodejs.org/api/stream.html#streams-promises-api)
 and stdlib interfaces like [`stream.pipeline`](https://nodejs.org/api/stream.html#streampipelinestreams-options).
 
-The main way to use `xml-stream-editor` is to 1. select which XML elements
-you want to edit using simple declarative selectors (like _very_ simple XPath
-rules or CSS selectors), and 2. write functions to be called with each
-matching XML element in the document. Those functions then either edit and
-return the provided element, or remove the element from the document
-by returning nothing.
+The main way to use `xml-stream-editor` is to:
 
-Example
----
+1. select which XML elements you want to edit using simple declarative selectors
+    (like _very_ simple XPath rules or CSS selectors), and
+2. write functions to be called with each matching XML element in the document.
+    Those functions then either edit and return the provided element, or remove
+    the element from the document by returning nothing.
+
+### Calling xml-stream-editor
+
+The main way to call `xml-stream-editor` is by importing `createXMLEditor`,
+passing that function an object, with keys as `selectors` (strings that describe
+which elements to edit) as keys, and values being functions that get passed
+matching elements (to edit to delete those elements).
+
+### Elements Selectors
+
+You choose which XML elements to edit by writing (simple, limited) CSS-selector
+like statements.  For example, the selector `parent child` will match
+all `<child>` elements that are _immediate_ children of `<parent>` nodes.
+**Note**, this is a little different than CSS selectors, where the selector
+`div a` would match `<a>` elements that were were contained in `<div>` elements,
+regardless of whether the `<a>` was an immediate child or more deeply nested.
+
+### Editing Elements
+
+Each element that matches a given selector is passed to the matching
+function, with the signature `(elm: Element) => Element | undefined`,
+and elements are structured as follows (as typescript):
+
+```typescript
+interface Element {
+  name: string
+  text?: string
+  attributes: Record<string, string>
+  children: Element[]
+}
+```
+
+### Options / Configuration
+
+In addition to a `rules` argument, `createReadStream` can also take
+a second `Options` argument. This object has the follow parameters.
+
+```typescript
+interface Options {
+  // Whether to check and enforce the validity of created and modified
+  // XML element names and attributes. If true, will throw an error
+  // if you create an XML element with a disallowed name (e.g.,
+  // <no spaces allowed>) or with an invalid attribute name
+  // (<my-elm a:b:c="too many namespaces" d@y="no @ in attr names">)
+  //
+  // This only checks the syntax of the XML element names and attributes.
+  // It does not perform any further validation, like if used namespaces
+  // are valid.
+  //
+  // default: `true`
+  validate: boolean // true
+
+  // Options defined by the "saxes" library, and passed to the "saxes" parser
+  //
+  // eslint-disable-next-line max-len
+  // https://github.com/lddubeau/saxes/blob/4968bd09b5fd0270a989c69913614b0e640dae1b/src/saxes.ts#L557
+  // https://www.npmjs.com/package/saxes
+  saxes?: SaxesOptions
+}
+
+// The createXMLEditor function takes the options object as an optional
+// second argument.
+const transformer = createXMLEditor(rules, options)
+```
+
+## Examples
 
 Start with this input as `simpsons.xml`:
 
@@ -55,8 +117,9 @@ import { pipeline } from 'node:stream/promises'
 import { createXMLEditor, newElement } from 'xml-stream-editor'
 
 (async () => {
-    const config = {
-        // Map element selectors to editing functions
+    // The keys of this object are selector strings, and the
+    // values are functions that get called with matching elements.
+    const rules = {
         "main character": (elm) => {
             switch (elm.text) {
                 case "Marge Simpson":
@@ -68,10 +131,14 @@ import { createXMLEditor, newElement } from 'xml-stream-editor'
                 case "Lisa Simpson":
                     elm.text = ""
 
+                    // Create an <instrument> element and make it
+                    // a child element.
                     const instrumentElm = newElement("instrument")
                     instrumentElm.text = "saxophone"
                     elm.children.push(instrumentElm)
 
+                    // Also create a new <name> element, and also make it
+                    // a child element.
                     const nameElm = newElement("name")
                     nameElm.text = "Lisa Simpson"
                     elm.children.push(nameElm)
@@ -85,18 +152,20 @@ import { createXMLEditor, newElement } from 'xml-stream-editor'
     }
     await pipeline(
         createReadStream("simpsons.xml"), // above example
-        createXMLEditor(config),
+        createXMLEditor(rules),
         process.stdout
     )
 })()
 ```
 
-And you'll find this printed to `STDOUT` (reformatted):
+And you'll find this printed to `STDOUT` (reformatted and annotated):
 
 ```xml
 <?xml version="1.0" encoding="UTF-8"?>
 <simpsons decade="90s" locale="US">
   <main>
+    <!-- These character elements were edited because they're
+         children of the main element (i.e., "main character"). -->
     <character sex="female" hair="blue">Marge Simpson</character>
     <character sex="male">Homer Simpson (Sr.)</character>
     <character sex="female">
@@ -104,16 +173,22 @@ And you'll find this printed to `STDOUT` (reformatted):
       <name>Lisa Simpson</name>
     </character>
     <character sex="female">Maggie Simpson</character>
+    <!-- There is no <character>Bart Simpson</character>
+         element anymore because the `case "Bart Simpson":`
+         case didn't return an element from the function. -->
   </main>
   <side>
+    <!-- These side character elements were not edited of affected
+         at all because they didn't match the given selector
+         (i.e., they are not "character" elements that are direct
+         children of "side" elements). -->
     <character sex="male">Disco Stu</character>
     <character sex="male" title="Dr.">Julius Hibbert</character>
   </side>
 </simpsons>
 ```
 
-Notes
----
+## Notes
 
 Nested editing functions are not supported. You can define as many editing
 rules as you'd like, but only one rule can be matching the xml document
@@ -129,7 +204,7 @@ import { pipeline } from 'node:stream/promises'
 import { createXMLEditor, newElement } from 'xml-stream-editor'
 
 (async () => {
-    const config = {
+    const rules = {
         // This rule will match first, since the "main" element will be
         // identified first during parsing.
         "main character": (elm) => {
@@ -147,14 +222,13 @@ import { createXMLEditor, newElement } from 'xml-stream-editor'
     }
     await pipeline(
         createReadStream("simpsons.xml"), // above example
-        createXMLEditor(config),
+        createXMLEditor(rules),
         process.stdout
     )
 })()
 ```
 
-Motivation
----
+## Motivation
 
 `xml-stream-editor` was built to handle the extremely large XML files
 generated by [Brave Software's PageGraph system](https://github.com/brave/brave-browser/wiki/PageGraph),

package/dist/markup.js

@@ -1,6 +1,6 @@
 import xmlescape from 'xml-escape';
 import xnv from 'xml-name-validator';
-export const isValidName = xnv.name;
+export const isValidName = xnv.qname;
 export const toAttrValue = (value) => {
     return xmlescape(value);
 };

package/dist/xml-stream-editor.js

@@ -3,16 +3,30 @@ import { Transform } from 'node:stream';
 import { SaxesParser } from 'saxes';
 import { isValidName, toAttrValue, toBodyText, toCloseTag, toOpenTag } from './markup.js';
 export const newElement = (name) => {
-    if (isValidName(name) === false) {
-        throw new Error(`"${name}" is not a valid XML element name`);
-    }
     return {
         name: name,
         text: undefined,
-        attributes: {},
+        attributes: Object.create(null),
         children: [],
     };
 };
+const throwOnInvalidElement = (elm) => {
+    if (typeof elm.name !== 'string') {
+        throw new Error('No name provided for element');
+    }
+    if (!isValidName(elm.name)) {
+        throw new Error(`"${elm.name}" is not a valid XML element name`);
+    }
+    if (typeof elm.attributes !== 'object' || elm.attributes === null) {
+        throw new Error('"attributes" property on element is not an object');
+    }
+    for (const attrName of Object.keys(elm.attributes)) {
+        if (!isValidName(attrName)) {
+            throw new Error(`"${attrName}" is not a valid XML attribute name`);
+        }
+    }
+    elm.children.forEach(throwOnInvalidElement);
+};
 const cloneElement = (elm) => {
     const newElm = newElement(elm.name);
     newElm.text = elm.text;
@@ -25,7 +39,7 @@ const elementForNode = (node) => {
     // string), or in the namespace representation the "saxes" library
     // uses (in which case attrValue will be a SaxesAttributeNS
     // object, that we have to unpack a bit)
-    const attributes = {};
+    const attributes = Object.create(null);
     if (node.attributes) {
         for (const [attrName, attrValue] of Object.entries(node.attributes)) {
             if (typeof attrValue === 'string') {
@@ -45,12 +59,21 @@ const elementForNameAndAttrs = (name, attrs) => {
     return newElm;
 };
 class XMLStreamEditorTransformer extends Transform {
+    // Default options, used if the caller doesn't provide any options (or
+    // merged into the provided options if the user only sets some options).
+    static defaultOptions = {
+        validate: true,
+        saxes: undefined,
+    };
+    // The configuration options, including possible options to pass to
+    // the (above) saxes parser at instantiation.
+    #options;
     // Used to track how deep in the XML tree the parser is, so that we can
     // check newly parsed elements against the passed editor rules.
-    #elmStack = [];
+    #parseStack = [];
     // This is a map of (VERY) simple xpaths (i.e., only XML element names;
     // no attributes, no name spaces, etc).
-    #config;
+    #editingRules;
     // Handle to the 'saxes' xml parser object.
     #xmlParser;
     // If set, tracks the current element in the parser stack that matches
@@ -60,17 +83,31 @@ class XMLStreamEditorTransformer extends Transform {
     // Store any errors we've been passed by the saxes parser so that we
     // can pass it along in the transformer callback next time we get data.
     #error;
+    #pushParsedElementToStack(element) {
+        const topOfStackElm = this.#parseStack.at(-1);
+        const pathToElement = topOfStackElm
+            ? topOfStackElm.path + ' ' + element.name
+            : element.name;
+        this.#parseStack.push({
+            element: element,
+            path: pathToElement,
+        });
+    }
     // Checks to see if the current editor stack (which tracks the current
     // element being parsed in the input XML stream, along with its parent
     // elements) matches any of the passed editor rules.
-    #doesStackMatchEditorRule() {
-        const currentElementPath = this.#elmStack.map(x => x.name).join(' ');
-        for (const [selector, editorFunc] of Object.entries(this.#config)) {
-            if (currentElementPath.endsWith(selector)) {
+    #doesStackMatchEditingRule() {
+        const topOfStack = this.#parseStack.at(-1);
+        // This method is only called after pushing an element to the stack,
+        // so this is guaranteed to be true
+        assert(topOfStack);
+        const topOfStackPath = topOfStack.path;
+        for (const [selector, editorFunc] of Object.entries(this.#editingRules)) {
+            if (topOfStackPath.endsWith(selector)) {
                 // The depth of the root of this subtree in the stack
-                const depth = this.#elmStack.length - 1;
+                const depth = this.#parseStack.length - 1;
                 assert(depth >= 0);
-                const elmToEdit = this.#elmStack[depth];
+                const elmToEdit = this.#parseStack[depth].element;
                 return { selector: selector, func: editorFunc, element: elmToEdit };
             }
         }
@@ -89,12 +126,15 @@ class XMLStreamEditorTransformer extends Transform {
         }
         this.push(toCloseTag(element.name));
     }
-    #writeSubtreeToStream() {
+    #callUserFuncOnCompletedElementAndWriteToStream() {
         assert(this.#elmToEditInfo);
         const clonedElm = cloneElement(this.#elmToEditInfo.element);
         try {
             const editedElm = this.#elmToEditInfo.func(clonedElm);
             if (editedElm) {
+                if (this.#options.validate === true) {
+                    throwOnInvalidElement(editedElm);
+                }
                 this.#writeElementToStream(editedElm);
             }
             this.#elmToEditInfo = undefined;
@@ -116,13 +156,13 @@ class XMLStreamEditorTransformer extends Transform {
             // 3. We are NOT the root of a subtree to be edited, in which case
             //    we just add ourselves to the stack.
             const newElement = elementForNode(node);
-            this.#elmStack.push(newElement);
+            this.#pushParsedElementToStack(newElement);
             // Check for case one
             if (this.#isInSubtreeToBeEdited()) {
                 return;
             }
             // Check for case two, if we're at the root of a subtree to edit.
-            const matchingElementInfo = this.#doesStackMatchEditorRule();
+            const matchingElementInfo = this.#doesStackMatchEditingRule();
             if (matchingElementInfo !== null) {
                 this.#elmToEditInfo = matchingElementInfo;
                 return;
@@ -140,9 +180,9 @@ class XMLStreamEditorTransformer extends Transform {
             //    print the text out immediately.
             // Check for case one
             if (this.#isInSubtreeToBeEdited()) {
-                const topOfStack = this.#elmStack.at(-1);
+                const topOfStack = this.#parseStack.at(-1);
                 assert(topOfStack);
-                topOfStack.text = text;
+                topOfStack.element.text = text;
                 return;
             }
             // Otherwise we're in case two, and can print the text out immediately.
@@ -176,36 +216,52 @@ class XMLStreamEditorTransformer extends Transform {
             // 3. We've completed a CHILD NODE in a subtree being edited,
             //    in which case we append this node to our buffered subtree
             //    and pop it off the stack.
-            const completedElm = this.#elmStack.pop();
+            const completedStackElement = this.#parseStack.pop();
+            const completedElm = completedStackElement?.element;
             assert(completedElm);
             // Check for case one
             if (this.#isInSubtreeToBeEdited() === false) {
+                // Write the closing tag of the just-completed element
+                // to the write stream.
                 this.push(toCloseTag(node.name));
                 return;
             }
             // Check for case two
             assert(this.#elmToEditInfo);
             if (completedElm === this.#elmToEditInfo.element) {
-                this.#writeSubtreeToStream();
+                this.#callUserFuncOnCompletedElementAndWriteToStream();
                 return;
             }
             // Otherwise, we must be in case three
-            assert(this.#elmStack.length > 0);
-            this.#elmStack.at(-1)?.children.push(completedElm);
+            const topOfStack = this.#parseStack.at(-1);
+            assert(topOfStack);
+            topOfStack.element.children.push(completedElm);
         });
     }
-    constructor(config, saxesOptions) {
+    constructor(editingRules, options) {
         super();
-        this.#config = config;
-        this.#xmlParser = new SaxesParser(saxesOptions);
+        const defaultOptions = XMLStreamEditorTransformer.defaultOptions;
+        const mergedOptions = {
+            validate: options?.validate ?? defaultOptions.validate,
+            saxes: options?.saxes ?? defaultOptions.saxes,
+        };
+        this.#options = mergedOptions;
+        this.#editingRules = editingRules;
+        this.#xmlParser = new SaxesParser(this.#options.saxes);
         this.#configureParserCallbacks();
     }
     _transform(chunk, encoding, callback) {
+        // Don't do any parsing if something threw an error parsing the previous
+        // chunk.
         if (this.#error) {
             callback(this.#error);
             return;
         }
         this.#xmlParser.write(chunk);
+        // And, similarly, don't continuing parsing if we've caught any errors
+        // parsing the current chunk. This looks a little redundant, but because
+        // the XML from the input stream is parsed asynchronously, this is
+        // just an attempt to catch and handle an error as quickly as possible.
         if (this.#error) {
             callback(this.#error);
             return;
@@ -213,6 +269,10 @@ class XMLStreamEditorTransformer extends Transform {
         callback();
     }
 }
-export const createXMLEditor = (config, saxesOptions) => {
-    return new XMLStreamEditorTransformer(config, saxesOptions);
+// This is the entry point to the library, and is designed / named
+// to mirror the naming of transformers in the standard lib
+// (e.g., createGzip , createDeflate, etc in the stdlib zlib module,
+// or createHmac, createECDH, etc in the stdlib crypto module).
+export const createXMLEditor = (rules, options) => {
+    return new XMLStreamEditorTransformer(rules, options);
 };

package/package.json

@@ -1,10 +1,11 @@
 {
   "name": "xml-stream-editor",
-  "version": "0.1.0",
+  "version": "0.2.0",
   "description": "A streaming xml editor.",
   "main": "dist/index.js",
   "files": [
     "dist/*",
+    "CHANGELOG.md",
     "src/types.d.ts"
   ],
   "scripts": {
@@ -17,7 +18,7 @@
   "type": "module",
   "types": "src/types.d.ts",
   "repository": {
-    "url": "https://github.com/pes10k/xml-stream-editor.git"
+    "url": "git+https://github.com/pes10k/xml-stream-editor.git"
   },
   "keywords": [
     "xml",

package/src/types.d.ts

@@ -9,9 +9,31 @@ export declare interface Element {
   children: Element[]
 }
 
+export declare interface Options {
+  // Whether to check and enforce the validity of created and modified
+  // XML element names and attributes. If true, will throw an error
+  // if you create an XML element with a disallowed name (e.g.,
+  // <no spaces allowed>) or with an invalid attribute name
+  // (<my-elm a:b:c="too many namespaces" d@y="no @ in attr names">)
+  //
+  // This only checks the syntax of the XML element names and attributes.
+  // It does not perform any further validation, like if used namespaces
+  // are valid.
+  //
+  // default: `true`
+  validate: boolean // true
+
+  // Options defined by the "saxes" library, and passed to the "saxes" parser
+  //
+  // eslint-disable-next-line max-len
+  // https://github.com/lddubeau/saxes/blob/4968bd09b5fd0270a989c69913614b0e640dae1b/src/saxes.ts#L557
+  // https://www.npmjs.com/package/saxes
+  saxes?: SaxesOptions
+}
+
 export type Selector = string
 export type EditorFunc = (elm: Element) => Element | undefined
-export type Config = Record<Selector, EditorFunc>
+export type EditingRules = Record<Selector, EditorFunc>
 export declare const newElement: (name: string) => Element
 export declare const createXMLEditor: (
-  config: Config, saxesOptions?: SaxesOptions) => Transform
+  editingRules: EditingRules, options?: Options) => Transform