sdf-parser 5.0.0 → 6.0.0

package/README.md

@@ -57,39 +57,18 @@ var result = parse(sdf, {
 });
 ```
 
-## Streams
+## Iterator
 
 This API is only available on Node.js.
 
-### molecules(options)
-
-Transform an input text stream to a stream of molecule objects.
-
-#### options
-
-- `fullResult`: true to emit the full result of `parse` instead of just the molecules.
-- All other options from the `parse` function.
-
-```js
-const { stream } = require('sdf-parser');
-fs.createReadStream('test.sdf')
-  .pipe(stream.molecules())
-  .on('data', (molecule) => {
-    console.log(molecule.molfile);
-  });
-```
-
-### entries()
-
-Transform an input text stream to a stream of sdf entries.
-
 ```js
-const { stream } = require('sdf-parser');
-fs.createReadStream('test.sdf')
-  .pipe(stream.entries())
-  .on('data', (entry) => {
-    // sdf entry as a string
-  });
+const { iterator } = require('sdf-parser');
+const readStream = createReadStream(join(__dirname, 'test.sdf.gz'));
+const stream = readStream.pipe(createGunzip());
+const results = [];
+for await (const entry of iterator(stream)) {
+  results.push(entry);
+}
 ```
 
 ## License
@@ -98,7 +77,7 @@ fs.createReadStream('test.sdf')
 
 [npm-image]: https://img.shields.io/npm/v/sdf-parser.svg?style=flat-square
 [npm-url]: https://www.npmjs.com/package/sdf-parser
-[travis-image]: https://img.shields.io/travis/cheminfo/sdf-parser/master.svg?style=flat-square
+[travis-image]: https://img.shields.io/travis/cheminfo/sdf-parser/main.svg?style=flat-square
 [travis-url]: https://travis-ci.org/cheminfo/sdf-parser
 [download-image]: https://img.shields.io/npm/dm/sdf-parser.svg?style=flat-square
 [download-url]: https://www.npmjs.com/package/sdf-parser

package/lib/index.js

@@ -2,17 +2,9 @@
 
 Object.defineProperty(exports, '__esModule', { value: true });
 
-var pipeline = require('pumpify');
-var split2 = require('split2');
-var through2 = require('through2');
-var filter = require('through2-filter');
-
-function _interopDefaultLegacy (e) { return e && typeof e === 'object' && 'default' in e ? e : { 'default': e }; }
-
-var pipeline__default = /*#__PURE__*/_interopDefaultLegacy(pipeline);
-var split2__default = /*#__PURE__*/_interopDefaultLegacy(split2);
-var through2__default = /*#__PURE__*/_interopDefaultLegacy(through2);
-var filter__default = /*#__PURE__*/_interopDefaultLegacy(filter);
+var ensureString = require('ensure-string');
+var readline = require('readline');
+var dynamicTyping = require('dynamic-typing');
 
 function getEntriesBoundaries(string, substring, eol) {
   const res = [];
@@ -22,8 +14,13 @@ function getEntriesBoundaries(string, substring, eol) {
     next = string.indexOf(substring, previous);
     if (next !== -1) {
       res.push([previous, next]);
-      previous = next =
-        string.indexOf(eol, next + substring.length) + eol.length;
+      const nextMatch = string.indexOf(eol, next + substring.length);
+      if (nextMatch === -1) {
+        next = -1;
+      } else {
+        previous = nextMatch + eol.length;
+        next = previous;
+      }
     } else {
       res.push([previous, string.length]);
     }
@@ -31,35 +28,106 @@ function getEntriesBoundaries(string, substring, eol) {
   return res;
 }
 
+function getMolecule$1(sdfPart, labels, currentLabels, options) {
+  let parts = sdfPart.split(`${options.eol}>`);
+  if (parts.length === 0 || parts[0].length <= 5) return;
+  let molecule = {};
+  molecule.molfile = parts[0] + options.eol;
+  for (let j = 1; j < parts.length; j++) {
+    let lines = parts[j].split(options.eol);
+    let from = lines[0].indexOf('<');
+    let to = lines[0].indexOf('>');
+    let label = lines[0].substring(from + 1, to);
+    currentLabels.push(label);
+    if (!labels[label]) {
+      labels[label] = {
+        counter: 0,
+        isNumeric: options.dynamicTyping,
+        keep: false,
+      };
+      if (
+        (!options.exclude || options.exclude.indexOf(label) === -1) &&
+        (!options.include || options.include.indexOf(label) > -1)
+      ) {
+        labels[label].keep = true;
+        if (options.modifiers[label]) {
+          labels[label].modifier = options.modifiers[label];
+        }
+        if (options.forEach[label]) {
+          labels[label].forEach = options.forEach[label];
+        }
+      }
+    }
+    if (labels[label].keep) {
+      for (let k = 1; k < lines.length - 1; k++) {
+        if (molecule[label]) {
+          molecule[label] += options.eol + lines[k];
+        } else {
+          molecule[label] = lines[k];
+        }
+      }
+      if (labels[label].modifier) {
+        let modifiedValue = labels[label].modifier(molecule[label]);
+        if (modifiedValue === undefined || modifiedValue === null) {
+          delete molecule[label];
+        } else {
+          molecule[label] = modifiedValue;
+        }
+      }
+      if (labels[label].isNumeric) {
+        if (!isFinite(molecule[label]) || molecule[label].match(/^0[0-9]/)) {
+          labels[label].isNumeric = false;
+        }
+      }
+    }
+  }
+  return molecule;
+}
+
+/**
+ *  Parse a SDF file
+ * @param {string|ArrayBuffer|Uint8Array} sdf SDF file to parse
+ * @param {object} [options={}]
+ * @param {string[]} [options.include] List of fields to include
+ * @param {string[]} [options.exclude] List of fields to exclude
+ * @param {Function} [options.filter] Callback allowing to filter the molecules
+ * @param {boolean} [options.dynamicTyping] Dynamically type the data
+ * @param {object} [options.modifiers] Object containing callbacks to apply on some specific fields
+ * @param {boolean} [options.mixedEOL=false] Set to true if you know there is a mixture between \r\n and \n
+ * @param {string} [options.eol] Specify the end of line character. Default will be the one found in the file
+ */
 function parse(sdf, options = {}) {
-  const {
-    include,
-    exclude,
-    filter,
-    modifiers = {},
-    forEach = {},
-    dynamicTyping = true,
-  } = options;
+  options = { ...options };
+  if (options.modifiers === undefined) options.modifiers = {};
+  if (options.forEach === undefined) options.forEach = {};
+  if (options.dynamicTyping === undefined) options.dynamicTyping = true;
 
+  sdf = ensureString.ensureString(sdf);
   if (typeof sdf !== 'string') {
     throw new TypeError('Parameter "sdf" must be a string');
   }
 
-  let eol = '\n';
-  if (options.mixedEOL) {
-    sdf = sdf.replace(/\r\n/g, '\n');
-    sdf = sdf.replace(/\r/g, '\n');
-  } else {
-    // we will find the delimiter in order to be much faster and not use regular expression
-    let header = sdf.substr(0, 1000);
-    if (header.indexOf('\r\n') > -1) {
-      eol = '\r\n';
-    } else if (header.indexOf('\r') > -1) {
-      eol = '\r';
+  if (options.eol === undefined) {
+    options.eol = '\n';
+    if (options.mixedEOL) {
+      sdf = sdf.replace(/\r\n/g, '\n');
+      sdf = sdf.replace(/\r/g, '\n');
+    } else {
+      // we will find the delimiter in order to be much faster and not use regular expression
+      let header = sdf.substr(0, 1000);
+      if (header.indexOf('\r\n') > -1) {
+        options.eol = '\r\n';
+      } else if (header.indexOf('\r') > -1) {
+        options.eol = '\r';
+      }
     }
   }
 
-  let entriesBoundaries = getEntriesBoundaries(sdf, `${eol}$$$$`, eol);
+  let entriesBoundaries = getEntriesBoundaries(
+    sdf,
+    `${options.eol}$$$$`,
+    options.eol,
+  );
   let molecules = [];
   let labels = {};
 
@@ -67,72 +135,18 @@ function parse(sdf, options = {}) {
 
   for (let i = 0; i < entriesBoundaries.length; i++) {
     let sdfPart = sdf.substring(...entriesBoundaries[i]);
-    let parts = sdfPart.split(`${eol}>`);
-    if (parts.length > 0 && parts[0].length > 5) {
-      let molecule = {};
-      let currentLabels = [];
-      molecule.molfile = parts[0] + eol;
-      for (let j = 1; j < parts.length; j++) {
-        let lines = parts[j].split(eol);
-        let from = lines[0].indexOf('<');
-        let to = lines[0].indexOf('>');
-        let label = lines[0].substring(from + 1, to);
-        currentLabels.push(label);
-        if (!labels[label]) {
-          labels[label] = {
-            counter: 0,
-            isNumeric: dynamicTyping,
-            keep: false,
-          };
-          if (
-            (!exclude || exclude.indexOf(label) === -1) &&
-            (!include || include.indexOf(label) > -1)
-          ) {
-            labels[label].keep = true;
-            if (modifiers[label]) {
-              labels[label].modifier = modifiers[label];
-            }
-            if (forEach[label]) {
-              labels[label].forEach = forEach[label];
-            }
-          }
-        }
-        if (labels[label].keep) {
-          for (let k = 1; k < lines.length - 1; k++) {
-            if (molecule[label]) {
-              molecule[label] += eol + lines[k];
-            } else {
-              molecule[label] = lines[k];
-            }
-          }
-          if (labels[label].modifier) {
-            let modifiedValue = labels[label].modifier(molecule[label]);
-            if (modifiedValue === undefined || modifiedValue === null) {
-              delete molecule[label];
-            } else {
-              molecule[label] = modifiedValue;
-            }
-          }
-          if (labels[label].isNumeric) {
-            if (
-              !isFinite(molecule[label]) ||
-              molecule[label].match(/^0[0-9]/)
-            ) {
-              labels[label].isNumeric = false;
-            }
-          }
-        }
-      }
-      if (!filter || filter(molecule)) {
-        molecules.push(molecule);
-        // only now we can increase the counter
-        for (let j = 0; j < currentLabels.length; j++) {
-          labels[currentLabels[j]].counter++;
-        }
+
+    let currentLabels = [];
+    const molecule = getMolecule$1(sdfPart, labels, currentLabels, options);
+    if (!molecule) continue;
+    if (!options.filter || options.filter(molecule)) {
+      molecules.push(molecule);
+      // only now we can increase the counter
+      for (let j = 0; j < currentLabels.length; j++) {
+        labels[currentLabels[j]].counter++;
       }
     }
   }
-
   // all numeric fields should be converted to numbers
   for (let label in labels) {
     let currentLabel = labels[label];
@@ -172,50 +186,63 @@ function parse(sdf, options = {}) {
 
   return {
     time: Date.now() - start,
-    molecules: molecules,
+    molecules,
     labels: Object.keys(labels),
-    statistics: statistics,
+    statistics,
   };
 }
 
-const filterStream = filter__default["default"].bind(null, { objectMode: true });
-function filterCb(chunk) {
-  return chunk.length > 1 && chunk.trim().length > 1;
-}
-
-function entries() {
-  return pipeline__default["default"].obj(
-    split2__default["default"](/\r?\n\${4}.*\r?\n/),
-    filterStream(filterCb),
-    through2__default["default"]({ objectMode: true }, function process(value, encoding, callback) {
-      const eol = value.includes('\r\n') ? '\r\n' : '\n';
-      this.push(`${value + eol}$$$$${eol}`);
-      callback();
-    }),
-  );
+/**
+ *  Parse a SDF file
+ * @param {string|ArrayBuffer|Uint8Array} sdf SDF file to parse
+ * @param {object} [options={}]
+ * @param {Function} [options.filter] Callback allowing to filter the molecules
+ * @param {boolean} [options.dynamicTyping] Dynamically type the data
+ */
+
+async function* iterator(readStream, options = {}) {
+  const lines = readline.createInterface(readStream);
+  const currentLines = [];
+  options = { ...options };
+  if (options.dynamicTyping === undefined) options.dynamicTyping = true;
+
+  options.eol = '\n';
+  for await (let line of lines) {
+    if (line.startsWith('$$$$')) {
+      const molecule = getMolecule(currentLines.join(options.eol), options);
+      if (!options.filter || options.filter(molecule)) {
+        yield molecule;
+      }
+      currentLines.length = 0;
+    } else {
+      currentLines.push(line);
+    }
+  }
 }
 
-function molecules(options) {
-  return pipeline__default["default"].obj(
-    entries(),
-    through2__default["default"]({ objectMode: true }, function process(value, encoding, callback) {
-      try {
-        const parsed = parse(value, options);
-        if (parsed.molecules.length === 1) {
-          if (options && options.fullResult) {
-            this.push(parsed);
-          } else {
-            this.push(parsed.molecules[0]);
-          }
-        }
-        callback();
-      } catch (e) {
-        callback(e);
+function getMolecule(sdfPart, options) {
+  let parts = sdfPart.split(`${options.eol}>`);
+  if (parts.length === 0 || parts[0].length <= 5) return;
+  let molecule = {};
+  molecule.molfile = parts[0] + options.eol;
+  for (let j = 1; j < parts.length; j++) {
+    let lines = parts[j].split(options.eol);
+    let from = lines[0].indexOf('<');
+    let to = lines[0].indexOf('>');
+    let label = lines[0].substring(from + 1, to);
+    for (let k = 1; k < lines.length - 1; k++) {
+      if (molecule[label]) {
+        molecule[label] += options.eol + lines[k];
+      } else {
+        molecule[label] = lines[k];
       }
-    }),
-  );
+    }
+    if (options.dynamicTyping) {
+      molecule[label] = dynamicTyping.parseString(molecule[label]);
+    }
+  }
+  return molecule;
 }
 
-exports.entries = entries;
-exports.molecules = molecules;
+exports.iterator = iterator;
 exports.parse = parse;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sdf-parser",
-  "version": "5.0.0",
+  "version": "6.0.0",
   "description": "SDF parser",
   "main": "lib/index.js",
   "module": "src/index.js",
@@ -8,6 +8,7 @@
     "lib",
     "src"
   ],
+  "sideEffects": false,
   "scripts": {
     "build": "npm run compile && cheminfo-build --root SDFParser",
     "compile": "rollup -c",
@@ -42,20 +43,19 @@
   },
   "homepage": "https://github.com/cheminfo/sdf-parser",
   "devDependencies": {
-    "@babel/plugin-transform-modules-commonjs": "^7.15.4",
+    "@babel/plugin-transform-modules-commonjs": "^7.18.6",
     "babel-eslint": "^10.1.0",
     "callback-stream": "^1.1.0",
     "cheminfo-build": "^1.1.11",
-    "eslint": "^7.32.0",
-    "eslint-config-cheminfo": "^6.0.1",
-    "jest": "^27.2.5",
-    "openchemlib": "^7.4.3",
-    "prettier": "^2.4.1"
+    "eslint": "^8.22.0",
+    "eslint-config-cheminfo": "^8.0.2",
+    "filelist-utils": "^0.6.0",
+    "jest": "^28.1.3",
+    "openchemlib": "^8.0.1",
+    "prettier": "^2.7.1"
   },
   "dependencies": {
-    "pumpify": "^2.0.1",
-    "split2": "^3.2.2",
-    "through2": "^4.0.2",
-    "through2-filter": "^3.0.0"
+    "dynamic-typing": "^1.0.0",
+    "ensure-string": "^1.2.0"
   }
 }

package/src/__tests__/__snapshots__/getEntriesBoundaries.test.js.snap

@@ -0,0 +1,10 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`should parse sdf files without EOL in the EOF 1`] = `
+Array [
+  Array [
+    0,
+    1473,
+  ],
+]
+`;

package/src/__tests__/getEntriesBoundaries.test.js

@@ -24,3 +24,10 @@ let sdf2 = fs.readFileSync(`${__dirname}/test2.sdf`, 'utf-8');
     );
   });
 });
+
+test('should parse sdf files without EOL in the EOF', () => {
+  const eol = '\n';
+  const sdf = fs.readFileSync(`${__dirname}/test4.sdf`, 'utf-8');
+
+  expect(getEntriesBoundaries(sdf, `${eol}$$$$`, eol)).toMatchSnapshot();
+});

package/src/__tests__/iterator.test.js

@@ -0,0 +1,120 @@
+import { createReadStream } from 'fs';
+import { join } from 'path';
+import { createGunzip } from 'zlib';
+
+import { fileListFromPath } from 'filelist-utils';
+
+import { iterator } from '../iterator';
+
+test('iterator', async () => {
+  const fileList = (await fileListFromPath(join(__dirname, '.'))).filter(
+    (file) => file.name === 'test.sdf',
+  );
+  const results = [];
+  for await (const entry of iterator(fileList[0].stream())) {
+    results.push(entry);
+  }
+  expect(results).toHaveLength(128);
+  expect(results[0]).toMatchInlineSnapshot(`
+    Object {
+      "CLogP": 2.7,
+      "Code": 100380824,
+      "Number of H-Acceptors": 3,
+      "Number of H-Donors": 1,
+      "Number of Rotatable bonds": 1,
+      "molfile": "
+      -ISIS-  04231216572D
+
+     15 16  0  0  0  0  0  0  0  0999 V2000
+        2.4792    1.7000    0.0000 N   0  0  0  0  0  0  0  0  0  0  0  0
+        2.4292    0.3500    0.0000 C   0  0  0  0  0  0  0  0  0  0  0  0
+        0.4042    1.1208    0.0000 C   0  0  0  0  0  0  0  0  0  0  0  0
+        1.2167    2.1833    0.0000 C   0  0  0  0  0  0  0  0  0  0  0  0
+        1.1542   -0.0000    0.0000 S   0  0  0  0  0  0  0  0  0  0  0  0
+       -0.9208    1.1208    0.0000 C   0  0  0  0  0  0  0  0  0  0  0  0
+        3.4792   -0.4500    0.0000 O   0  0  0  0  0  0  0  0  0  0  0  0
+        0.8792    3.4458    0.0000 O   0  0  0  0  0  0  0  0  0  0  0  0
+       -1.6000   -0.0292    0.0000 C   0  0  0  0  0  0  0  0  0  0  0  0
+       -0.9625   -1.1792    0.0000 C   0  0  0  0  0  0  0  0  0  0  0  0
+       -1.6208   -2.3292    0.0000 C   0  0  0  0  0  0  0  0  0  0  0  0
+       -0.9125   -3.4375    0.0000 Br  0  0  0  0  0  0  0  0  0  0  0  0
+       -3.5958   -1.1792    0.0000 C   0  0  0  0  0  0  0  0  0  0  0  0
+       -2.9208   -0.0292    0.0000 C   0  0  0  0  0  0  0  0  0  0  0  0
+       -3.0333   -2.3292    0.0000 C   0  0  0  0  0  0  0  0  0  0  0  0
+      2  1  1  0  0  0  0
+      3  4  1  0  0  0  0
+      4  1  1  0  0  0  0
+      5  2  1  0  0  0  0
+      6  3  2  0  0  0  0
+      7  2  2  0  0  0  0
+      8  4  2  0  0  0  0
+      9  6  1  0  0  0  0
+     10  9  2  0  0  0  0
+     11 10  1  0  0  0  0
+     12 11  1  0  0  0  0
+     13 14  2  0  0  0  0
+     14  9  1  0  0  0  0
+     15 13  1  0  0  0  0
+      3  5  1  0  0  0  0
+     15 11  2  0  0  0  0
+    M  END
+    ",
+    }
+  `);
+});
+
+test('iterator on stream', async () => {
+  const readStream = createReadStream(join(__dirname, 'test.sdf.gz'));
+  const stream = readStream.pipe(createGunzip());
+  const results = [];
+  for await (const entry of iterator(stream)) {
+    results.push(entry);
+  }
+  expect(results).toHaveLength(128);
+  expect(results[0]).toMatchInlineSnapshot(`
+    Object {
+      "CLogP": 2.7,
+      "Code": 100380824,
+      "Number of H-Acceptors": 3,
+      "Number of H-Donors": 1,
+      "Number of Rotatable bonds": 1,
+      "molfile": "
+      -ISIS-  04231216572D
+
+     15 16  0  0  0  0  0  0  0  0999 V2000
+        2.4792    1.7000    0.0000 N   0  0  0  0  0  0  0  0  0  0  0  0
+        2.4292    0.3500    0.0000 C   0  0  0  0  0  0  0  0  0  0  0  0
+        0.4042    1.1208    0.0000 C   0  0  0  0  0  0  0  0  0  0  0  0
+        1.2167    2.1833    0.0000 C   0  0  0  0  0  0  0  0  0  0  0  0
+        1.1542   -0.0000    0.0000 S   0  0  0  0  0  0  0  0  0  0  0  0
+       -0.9208    1.1208    0.0000 C   0  0  0  0  0  0  0  0  0  0  0  0
+        3.4792   -0.4500    0.0000 O   0  0  0  0  0  0  0  0  0  0  0  0
+        0.8792    3.4458    0.0000 O   0  0  0  0  0  0  0  0  0  0  0  0
+       -1.6000   -0.0292    0.0000 C   0  0  0  0  0  0  0  0  0  0  0  0
+       -0.9625   -1.1792    0.0000 C   0  0  0  0  0  0  0  0  0  0  0  0
+       -1.6208   -2.3292    0.0000 C   0  0  0  0  0  0  0  0  0  0  0  0
+       -0.9125   -3.4375    0.0000 Br  0  0  0  0  0  0  0  0  0  0  0  0
+       -3.5958   -1.1792    0.0000 C   0  0  0  0  0  0  0  0  0  0  0  0
+       -2.9208   -0.0292    0.0000 C   0  0  0  0  0  0  0  0  0  0  0  0
+       -3.0333   -2.3292    0.0000 C   0  0  0  0  0  0  0  0  0  0  0  0
+      2  1  1  0  0  0  0
+      3  4  1  0  0  0  0
+      4  1  1  0  0  0  0
+      5  2  1  0  0  0  0
+      6  3  2  0  0  0  0
+      7  2  2  0  0  0  0
+      8  4  2  0  0  0  0
+      9  6  1  0  0  0  0
+     10  9  2  0  0  0  0
+     11 10  1  0  0  0  0
+     12 11  1  0  0  0  0
+     13 14  2  0  0  0  0
+     14  9  1  0  0  0  0
+     15 13  1  0  0  0  0
+      3  5  1  0  0  0  0
+     15 11  2  0  0  0  0
+    M  END
+    ",
+    }
+  `);
+});

package/src/__tests__/test4.sdf

@@ -0,0 +1,37 @@
+
+  -ISIS-  04231216572D
+
+ 15 16  0  0  0  0  0  0  0  0999 V2000
+    2.4792    1.7000    0.0000 N   0  0  0  0  0  0  0  0  0  0  0  0
+    2.4292    0.3500    0.0000 C   0  0  0  0  0  0  0  0  0  0  0  0
+    0.4042    1.1208    0.0000 C   0  0  0  0  0  0  0  0  0  0  0  0
+    1.2167    2.1833    0.0000 C   0  0  0  0  0  0  0  0  0  0  0  0
+    1.1542   -0.0000    0.0000 S   0  0  0  0  0  0  0  0  0  0  0  0
+   -0.9208    1.1208    0.0000 C   0  0  0  0  0  0  0  0  0  0  0  0
+    3.4792   -0.4500    0.0000 O   0  0  0  0  0  0  0  0  0  0  0  0
+    0.8792    3.4458    0.0000 O   0  0  0  0  0  0  0  0  0  0  0  0
+   -1.6000   -0.0292    0.0000 C   0  0  0  0  0  0  0  0  0  0  0  0
+   -0.9625   -1.1792    0.0000 C   0  0  0  0  0  0  0  0  0  0  0  0
+   -1.6208   -2.3292    0.0000 C   0  0  0  0  0  0  0  0  0  0  0  0
+   -0.9125   -3.4375    0.0000 Br  0  0  0  0  0  0  0  0  0  0  0  0
+   -3.5958   -1.1792    0.0000 C   0  0  0  0  0  0  0  0  0  0  0  0
+   -2.9208   -0.0292    0.0000 C   0  0  0  0  0  0  0  0  0  0  0  0
+   -3.0333   -2.3292    0.0000 C   0  0  0  0  0  0  0  0  0  0  0  0
+  2  1  1  0  0  0  0
+  3  4  1  0  0  0  0
+  4  1  1  0  0  0  0
+  5  2  1  0  0  0  0
+  6  3  2  0  0  0  0
+  7  2  2  0  0  0  0
+  8  4  2  0  0  0  0
+  9  6  1  0  0  0  0
+ 10  9  2  0  0  0  0
+ 11 10  1  0  0  0  0
+ 12 11  1  0  0  0  0
+ 13 14  2  0  0  0  0
+ 14  9  1  0  0  0  0
+ 15 13  1  0  0  0  0
+  3  5  1  0  0  0  0
+ 15 11  2  0  0  0  0
+M  END
+$$$$

package/src/getEntriesBoundaries.js

@@ -6,8 +6,13 @@ export function getEntriesBoundaries(string, substring, eol) {
     next = string.indexOf(substring, previous);
     if (next !== -1) {
       res.push([previous, next]);
-      previous = next =
-        string.indexOf(eol, next + substring.length) + eol.length;
+      const nextMatch = string.indexOf(eol, next + substring.length);
+      if (nextMatch === -1) {
+        next = -1;
+      } else {
+        previous = nextMatch + eol.length;
+        next = previous;
+      }
     } else {
       res.push([previous, string.length]);
     }

package/src/index.js

@@ -1,2 +1,2 @@
 export * from './parse';
-export * from './stream';
+export * from './iterator';

package/src/iterator.js

@@ -0,0 +1,54 @@
+import { createInterface } from 'readline';
+
+import { parseString } from 'dynamic-typing';
+/**
+ *  Parse a SDF file
+ * @param {string|ArrayBuffer|Uint8Array} sdf SDF file to parse
+ * @param {object} [options={}]
+ * @param {Function} [options.filter] Callback allowing to filter the molecules
+ * @param {boolean} [options.dynamicTyping] Dynamically type the data
+ */
+
+export async function* iterator(readStream, options = {}) {
+  const lines = createInterface(readStream);
+  const currentLines = [];
+  options = { ...options };
+  if (options.dynamicTyping === undefined) options.dynamicTyping = true;
+
+  options.eol = '\n';
+  for await (let line of lines) {
+    if (line.startsWith('$$$$')) {
+      const molecule = getMolecule(currentLines.join(options.eol), options);
+      if (!options.filter || options.filter(molecule)) {
+        yield molecule;
+      }
+      currentLines.length = 0;
+    } else {
+      currentLines.push(line);
+    }
+  }
+}
+
+function getMolecule(sdfPart, options) {
+  let parts = sdfPart.split(`${options.eol}>`);
+  if (parts.length === 0 || parts[0].length <= 5) return;
+  let molecule = {};
+  molecule.molfile = parts[0] + options.eol;
+  for (let j = 1; j < parts.length; j++) {
+    let lines = parts[j].split(options.eol);
+    let from = lines[0].indexOf('<');
+    let to = lines[0].indexOf('>');
+    let label = lines[0].substring(from + 1, to);
+    for (let k = 1; k < lines.length - 1; k++) {
+      if (molecule[label]) {
+        molecule[label] += options.eol + lines[k];
+      } else {
+        molecule[label] = lines[k];
+      }
+    }
+    if (options.dynamicTyping) {
+      molecule[label] = parseString(molecule[label]);
+    }
+  }
+  return molecule;
+}

package/src/parse.js

@@ -1,34 +1,51 @@
-import { getEntriesBoundaries } from './getEntriesBoundaries';
+import { ensureString } from 'ensure-string';
 
+import { getEntriesBoundaries } from './getEntriesBoundaries';
+import { getMolecule } from './util/getMolecule';
+/**
+ *  Parse a SDF file
+ * @param {string|ArrayBuffer|Uint8Array} sdf SDF file to parse
+ * @param {object} [options={}]
+ * @param {string[]} [options.include] List of fields to include
+ * @param {string[]} [options.exclude] List of fields to exclude
+ * @param {Function} [options.filter] Callback allowing to filter the molecules
+ * @param {boolean} [options.dynamicTyping] Dynamically type the data
+ * @param {object} [options.modifiers] Object containing callbacks to apply on some specific fields
+ * @param {boolean} [options.mixedEOL=false] Set to true if you know there is a mixture between \r\n and \n
+ * @param {string} [options.eol] Specify the end of line character. Default will be the one found in the file
+ */
 export function parse(sdf, options = {}) {
-  const {
-    include,
-    exclude,
-    filter,
-    modifiers = {},
-    forEach = {},
-    dynamicTyping = true,
-  } = options;
+  options = { ...options };
+  if (options.modifiers === undefined) options.modifiers = {};
+  if (options.forEach === undefined) options.forEach = {};
+  if (options.dynamicTyping === undefined) options.dynamicTyping = true;
 
+  sdf = ensureString(sdf);
   if (typeof sdf !== 'string') {
     throw new TypeError('Parameter "sdf" must be a string');
   }
 
-  let eol = '\n';
-  if (options.mixedEOL) {
-    sdf = sdf.replace(/\r\n/g, '\n');
-    sdf = sdf.replace(/\r/g, '\n');
-  } else {
-    // we will find the delimiter in order to be much faster and not use regular expression
-    let header = sdf.substr(0, 1000);
-    if (header.indexOf('\r\n') > -1) {
-      eol = '\r\n';
-    } else if (header.indexOf('\r') > -1) {
-      eol = '\r';
+  if (options.eol === undefined) {
+    options.eol = '\n';
+    if (options.mixedEOL) {
+      sdf = sdf.replace(/\r\n/g, '\n');
+      sdf = sdf.replace(/\r/g, '\n');
+    } else {
+      // we will find the delimiter in order to be much faster and not use regular expression
+      let header = sdf.substr(0, 1000);
+      if (header.indexOf('\r\n') > -1) {
+        options.eol = '\r\n';
+      } else if (header.indexOf('\r') > -1) {
+        options.eol = '\r';
+      }
     }
   }
 
-  let entriesBoundaries = getEntriesBoundaries(sdf, `${eol}$$$$`, eol);
+  let entriesBoundaries = getEntriesBoundaries(
+    sdf,
+    `${options.eol}$$$$`,
+    options.eol,
+  );
   let molecules = [];
   let labels = {};
 
@@ -36,72 +53,18 @@ export function parse(sdf, options = {}) {
 
   for (let i = 0; i < entriesBoundaries.length; i++) {
     let sdfPart = sdf.substring(...entriesBoundaries[i]);
-    let parts = sdfPart.split(`${eol}>`);
-    if (parts.length > 0 && parts[0].length > 5) {
-      let molecule = {};
-      let currentLabels = [];
-      molecule.molfile = parts[0] + eol;
-      for (let j = 1; j < parts.length; j++) {
-        let lines = parts[j].split(eol);
-        let from = lines[0].indexOf('<');
-        let to = lines[0].indexOf('>');
-        let label = lines[0].substring(from + 1, to);
-        currentLabels.push(label);
-        if (!labels[label]) {
-          labels[label] = {
-            counter: 0,
-            isNumeric: dynamicTyping,
-            keep: false,
-          };
-          if (
-            (!exclude || exclude.indexOf(label) === -1) &&
-            (!include || include.indexOf(label) > -1)
-          ) {
-            labels[label].keep = true;
-            if (modifiers[label]) {
-              labels[label].modifier = modifiers[label];
-            }
-            if (forEach[label]) {
-              labels[label].forEach = forEach[label];
-            }
-          }
-        }
-        if (labels[label].keep) {
-          for (let k = 1; k < lines.length - 1; k++) {
-            if (molecule[label]) {
-              molecule[label] += eol + lines[k];
-            } else {
-              molecule[label] = lines[k];
-            }
-          }
-          if (labels[label].modifier) {
-            let modifiedValue = labels[label].modifier(molecule[label]);
-            if (modifiedValue === undefined || modifiedValue === null) {
-              delete molecule[label];
-            } else {
-              molecule[label] = modifiedValue;
-            }
-          }
-          if (labels[label].isNumeric) {
-            if (
-              !isFinite(molecule[label]) ||
-              molecule[label].match(/^0[0-9]/)
-            ) {
-              labels[label].isNumeric = false;
-            }
-          }
-        }
-      }
-      if (!filter || filter(molecule)) {
-        molecules.push(molecule);
-        // only now we can increase the counter
-        for (let j = 0; j < currentLabels.length; j++) {
-          labels[currentLabels[j]].counter++;
-        }
+
+    let currentLabels = [];
+    const molecule = getMolecule(sdfPart, labels, currentLabels, options);
+    if (!molecule) continue;
+    if (!options.filter || options.filter(molecule)) {
+      molecules.push(molecule);
+      // only now we can increase the counter
+      for (let j = 0; j < currentLabels.length; j++) {
+        labels[currentLabels[j]].counter++;
       }
     }
   }
-
   // all numeric fields should be converted to numbers
   for (let label in labels) {
     let currentLabel = labels[label];
@@ -141,8 +104,8 @@ export function parse(sdf, options = {}) {
 
   return {
     time: Date.now() - start,
-    molecules: molecules,
+    molecules,
     labels: Object.keys(labels),
-    statistics: statistics,
+    statistics,
   };
 }

package/src/util/getMolecule.js

@@ -0,0 +1,55 @@
+export function getMolecule(sdfPart, labels, currentLabels, options) {
+  let parts = sdfPart.split(`${options.eol}>`);
+  if (parts.length === 0 || parts[0].length <= 5) return;
+  let molecule = {};
+  molecule.molfile = parts[0] + options.eol;
+  for (let j = 1; j < parts.length; j++) {
+    let lines = parts[j].split(options.eol);
+    let from = lines[0].indexOf('<');
+    let to = lines[0].indexOf('>');
+    let label = lines[0].substring(from + 1, to);
+    currentLabels.push(label);
+    if (!labels[label]) {
+      labels[label] = {
+        counter: 0,
+        isNumeric: options.dynamicTyping,
+        keep: false,
+      };
+      if (
+        (!options.exclude || options.exclude.indexOf(label) === -1) &&
+        (!options.include || options.include.indexOf(label) > -1)
+      ) {
+        labels[label].keep = true;
+        if (options.modifiers[label]) {
+          labels[label].modifier = options.modifiers[label];
+        }
+        if (options.forEach[label]) {
+          labels[label].forEach = options.forEach[label];
+        }
+      }
+    }
+    if (labels[label].keep) {
+      for (let k = 1; k < lines.length - 1; k++) {
+        if (molecule[label]) {
+          molecule[label] += options.eol + lines[k];
+        } else {
+          molecule[label] = lines[k];
+        }
+      }
+      if (labels[label].modifier) {
+        let modifiedValue = labels[label].modifier(molecule[label]);
+        if (modifiedValue === undefined || modifiedValue === null) {
+          delete molecule[label];
+        } else {
+          molecule[label] = modifiedValue;
+        }
+      }
+      if (labels[label].isNumeric) {
+        if (!isFinite(molecule[label]) || molecule[label].match(/^0[0-9]/)) {
+          labels[label].isNumeric = false;
+        }
+      }
+    }
+  }
+  return molecule;
+}

package/src/__tests__/stream.test.js

@@ -1,98 +0,0 @@
-import fs from 'fs';
-
-import callbackStream from 'callback-stream';
-import OCL from 'openchemlib/minimal';
-
-import { entries, molecules } from '..';
-
-const cbStream = callbackStream.bind(null, { objectMode: true });
-
-describe('stream', () => {
-  it('entries', () =>
-    new Promise((resolve) => {
-      fs.createReadStream(`${__dirname}/test.sdf`)
-        .pipe(entries())
-        .pipe(
-          cbStream((err, data) => {
-            expect(err).toBeNull();
-            expect(data).toHaveLength(128);
-            expect(data[0]).toContain('-ISIS-  04231216572D');
-            const mol = OCL.Molecule.fromMolfile(data[5]);
-            expect(mol.toMolfile()).toContain(
-              '17 18  0  0  0  0  0  0  0  0999 V2000',
-            );
-            resolve();
-          }),
-        );
-    }));
-
-  it('molecules', () =>
-    new Promise((resolve) => {
-      fs.createReadStream(`${__dirname}/test.sdf`)
-        .pipe(molecules())
-        .pipe(
-          cbStream((err, data) => {
-            expect(err).toBeNull();
-            expect(data).toHaveLength(128);
-            expect(data[0]).toMatchObject({
-              Code: '0100380824',
-              CLogP: 2.7,
-            });
-            expect(data[0].molfile).toContain('-ISIS-  04231216572D');
-            resolve();
-          }),
-        );
-    }));
-
-  it('molecules - full result', () =>
-    new Promise((resolve) => {
-      fs.createReadStream(`${__dirname}/test.sdf`)
-        .pipe(molecules({ fullResult: true }))
-        .pipe(
-          cbStream((err, data) => {
-            expect(err).toBeNull();
-            expect(data).toHaveLength(128);
-            expect(data[0]).toMatchObject({
-              labels: [
-                'Code',
-                'Number of H-Donors',
-                'Number of H-Acceptors',
-                'Number of Rotatable bonds',
-                'CLogP',
-              ],
-            });
-            expect(data[0].molecules).toHaveLength(1);
-            resolve();
-          }),
-        );
-    }));
-
-  it('molecules with filter', () =>
-    new Promise((resolve) => {
-      fs.createReadStream(`${__dirname}/test.sdf`)
-        .pipe(
-          molecules({
-            filter: (entry) => entry.Code === '0100380869',
-          }),
-        )
-        .pipe(
-          cbStream((err, data) => {
-            expect(err).toBeNull();
-            expect(data).toHaveLength(1);
-            resolve();
-          }),
-        );
-    }));
-
-  it('async iteration', async () => {
-    const stream = fs
-      .createReadStream(`${__dirname}/test.sdf`)
-      .pipe(molecules());
-    let count = 0;
-    for await (const molecule of stream) {
-      count++;
-      expect(molecule.molfile.toString()).toContain('0999 V2000');
-    }
-    expect(count).toBe(128);
-  });
-});

package/src/stream.js

@@ -1,44 +0,0 @@
-import pipeline from 'pumpify';
-import split2 from 'split2';
-import through2 from 'through2';
-import filter from 'through2-filter';
-
-import { parse } from './parse';
-
-const filterStream = filter.bind(null, { objectMode: true });
-function filterCb(chunk) {
-  return chunk.length > 1 && chunk.trim().length > 1;
-}
-
-export function entries() {
-  return pipeline.obj(
-    split2(/\r?\n\${4}.*\r?\n/),
-    filterStream(filterCb),
-    through2({ objectMode: true }, function process(value, encoding, callback) {
-      const eol = value.includes('\r\n') ? '\r\n' : '\n';
-      this.push(`${value + eol}$$$$${eol}`);
-      callback();
-    }),
-  );
-}
-
-export function molecules(options) {
-  return pipeline.obj(
-    entries(),
-    through2({ objectMode: true }, function process(value, encoding, callback) {
-      try {
-        const parsed = parse(value, options);
-        if (parsed.molecules.length === 1) {
-          if (options && options.fullResult) {
-            this.push(parsed);
-          } else {
-            this.push(parsed.molecules[0]);
-          }
-        }
-        callback();
-      } catch (e) {
-        callback(e);
-      }
-    }),
-  );
-}