sitemap-xml-parser 1.1.0 → 1.3.0

package/README.md

@@ -8,6 +8,68 @@ Parses sitemap XML files and returns all listed URLs. Supports sitemap index fil
 npm install sitemap-xml-parser
 ```
 
+## CLI
+
+Run without installing via `npx`:
+
+```sh
+npx sitemap-xml-parser <url> [options]
+```
+
+Or, after installing globally (`npm install -g sitemap-xml-parser`):
+
+```sh
+sitemap-xml-parser <url> [options]
+```
+
+Fetched URLs are printed to stdout, one per line. Errors are printed to stderr. See [Options](#options) for available flags.
+
+## Examples
+
+```sh
+# Print all URLs
+npx sitemap-xml-parser https://example.com/sitemap.xml
+
+# Count URLs
+npx sitemap-xml-parser https://example.com/sitemap.xml --count
+
+# Filter by substring
+npx sitemap-xml-parser https://example.com/sitemap.xml --filter "blog"
+
+# Filter by regular expression
+npx sitemap-xml-parser https://example.com/sitemap.xml --filter-regex "blog/[0-9]{4}/"
+
+# Filter and count
+npx sitemap-xml-parser https://example.com/sitemap.xml --filter "blog" --count
+
+# Output as TSV
+npx sitemap-xml-parser https://example.com/sitemap.xml --tsv > urls.tsv
+
+# Save URLs to a file, errors to a log
+npx sitemap-xml-parser https://example.com/sitemap.xml > urls.txt 2> errors.log
+```
+
+## Options
+
+| Option              | Type       | Default | Description                                                                 |
+|---------------------|------------|---------|-----------------------------------------------------------------------------|
+| `delay`             | `number`   | `1000`  | Milliseconds to wait between batches when following a sitemap index. `limit` URLs are fetched in parallel per batch; after each batch completes, the process waits `delay` ms before starting the next. Set to `0` to disable. CLI: `--delay`   |
+| `limit`             | `number`   | `10`    | Number of child sitemaps to fetch concurrently per batch. CLI: `--limit`              |
+| `timeout`           | `number`   | `30000` | Milliseconds before a request is aborted. CLI: `--timeout`                            |
+| `onError`           | `function` | —       | Called as `onError(url, error)` when a URL fails. The URL is skipped regardless. **Library only.** |
+| `onEntry`           | `function` | —       | Called as `onEntry(entry)` each time a URL entry is parsed. `entry` has the same shape as the objects returned by `fetch()`. **Library only.** |
+| `filter`            | `string`   | —       | Only output URLs whose `loc` contains the given string (substring match). Can be combined with `--count` or `--tsv`. **CLI only.** |
+| `filter-regex`      | `string`   | —       | Only output URLs whose `loc` matches the given regular expression (evaluated with `new RegExp(value)`). Invalid patterns exit with a non-zero code and an error on stderr. Can be combined with `--count` or `--tsv`. **CLI only.** |
+| `tsv`               | —          | —       | Output results as tab-separated values. Prints a header row (`loc`, `lastmod`, `changefreq`, `priority`) followed by one row per entry. Missing fields are output as empty strings. **CLI only.** |
+| `count`             | —          | —       | Print only the total number of URLs instead of listing them. **CLI only.** |
+
+## Features
+
+- Follows Sitemap Index files recursively, including nested indexes (Index within an Index)
+- Automatically decompresses gzip: supports both `.gz` URLs and `Content-Encoding: gzip` responses
+- Batch processing: fetches `limit` child sitemaps in parallel per batch, then waits `delay` ms after each batch completes
+- Automatically follows redirects (301/302/303/307/308) up to 5 hops; errors beyond that are reported via `onError`
+
 ## Usage
 
 ```js
@@ -18,11 +80,24 @@ const parser = new SitemapXMLParser('https://example.com/sitemap.xml');
 (async () => {
     const urls = await parser.fetch();
     urls.forEach(entry => {
-        console.log(entry.loc[0]);
+        console.log(entry.loc);
     });
 })();
 ```
 
+Or with ES modules:
+
+```js
+import SitemapXMLParser from 'sitemap-xml-parser';
+
+const parser = new SitemapXMLParser('https://example.com/sitemap.xml');
+
+const urls = await parser.fetch();
+urls.forEach(entry => {
+    console.log(entry.loc);
+});
+```
+
 ### Error handling with `onError`
 
 Failed URLs (network errors, non-2xx responses, malformed XML) are skipped by default. Provide an `onError` callback to inspect them:
@@ -35,17 +110,6 @@ const parser = new SitemapXMLParser('https://example.com/sitemap.xml', {
 });
 ```
 
-## Options
-
-| Option      | Type       | Default | Description                                                                 |
-|-------------|------------|---------|-----------------------------------------------------------------------------|
-| `delay`     | `number`   | `3000`  | Milliseconds to wait between batches when following a sitemap index. Default is 3000 to avoid overloading the target server; set to `0` to disable. CLI: `--delay`   |
-| `limit`     | `number`   | `5`     | Number of child sitemaps to fetch concurrently per batch. CLI: `--limit`              |
-| `timeout`   | `number`   | `30000` | Milliseconds before a request is aborted. CLI: `--timeout`                            |
-| `onError`   | `function` | —       | Called as `onError(url, error)` when a URL fails. The URL is skipped regardless. **Library only.** |
-| `onEntry`   | `function` | —       | Called as `onEntry(entry)` each time a URL entry is parsed. `entry` has the same shape as the objects returned by `fetch()`. **Library only.** |
-| `tsv`     | —          | —       | Output results as tab-separated values. Prints a header row (`loc`, `lastmod`, `changefreq`, `priority`) followed by one row per entry. Missing fields are output as empty strings. **CLI only.** |
-
 ## Return value
 
 `fetch()` resolves to an array of URL entry objects. Each object reflects the fields present in the sitemap:
@@ -53,57 +117,16 @@ const parser = new SitemapXMLParser('https://example.com/sitemap.xml', {
 ```js
 [
   {
-    loc:        ['https://example.com/page1'],
-    lastmod:    ['2024-01-01'],
-    changefreq: ['weekly'],
-    priority:   ['0.8'],
+    loc:        'https://example.com/page1',
+    lastmod:    '2024-01-01',
+    changefreq: 'weekly',
+    priority:   '0.8',
   },
   // ...
 ]
 ```
 
-All field values are arrays (xml2js convention). Use `entry.loc[0]` to get the URL string, `entry.lastmod?.[0]` for optional fields, and so on.
+`loc` is always a string. Use `entry.loc` to get the URL. Optional fields (`lastmod`, `changefreq`, `priority`) are strings when present, or `undefined` when absent from the source XML.
 
 Fields other than `loc` (`lastmod`, `changefreq`, `priority`, etc.) are included only when present in the source XML.
 
-## CLI
-
-Run without installing via `npx`:
-
-```sh
-npx sitemap-xml-parser <url> [options]
-```
-
-Or, after installing globally (`npm install -g sitemap-xml-parser`):
-
-```sh
-sitemap-xml-parser <url> [options]
-```
-
-Fetched URLs are printed to stdout, one per line. Errors are printed to stderr. See [Options](#options) for available flags.
-
-### Examples
-
-```sh
-# Print all URLs
-npx sitemap-xml-parser https://example.com/sitemap.xml
-
-# No delay, higher concurrency
-npx sitemap-xml-parser https://example.com/sitemap.xml --delay 0 --limit 10
-
-# Save URLs to a file, errors to a log
-npx sitemap-xml-parser https://example.com/sitemap.xml > urls.txt 2> errors.log
-
-# Custom timeout
-npx sitemap-xml-parser https://example.com/sitemap.xml --timeout 10000
-
-# Output as TSV (includes lastmod, changefreq, priority)
-npx sitemap-xml-parser https://example.com/sitemap.xml --tsv
-
-# Save TSV to a file
-npx sitemap-xml-parser https://example.com/sitemap.xml --tsv > urls.tsv
-```
-
-## Limitations
-
-- **HTTP redirects are followed up to 5 times.** Status codes 301, 302, 303, 307, and 308 are handled automatically by following the `Location` header (relative URLs are resolved against the current URL). If the redirect chain exceeds 5 hops, an error is raised via `onError`.

package/bin/cli.js

@@ -8,20 +8,26 @@ function printUsage() {
         'Usage: sitemap-xml-parser <url> [options]',
         '',
         'Options:',
-        '  --delay <ms>    Delay between batches in milliseconds (default: 3000)',
-        '  --limit <n>     Concurrent fetches per batch (default: 5)',
-        '  --timeout <ms>  Request timeout in milliseconds (default: 30000)',
-        '  --tsv           Output as tab-separated values with a header row',
-        '  --help          Show this help message',
+        '  --delay <ms>           Delay between batches in milliseconds (default: 1000)',
+        '  --limit <n>            Concurrent fetches per batch (default: 10)',
+        '  --timeout <ms>         Request timeout in milliseconds (default: 30000)',
+        '  --filter <str>         Only output URLs that contain <str>',
+        '  --filter-regex <regex> Only output URLs matching the given regular expression',
+        '  --tsv                  Output as tab-separated values with a header row',
+        '  --count                Print only the total number of URLs',
+        '  --help                 Show this help message',
         '',
     ].join('\n'));
 }
 
 function parseArgs(argv) {
     const args = argv.slice(2);
-    const opts = { delay: 3000, limit: 5, timeout: 30000 };
+    const opts = { delay: 1000, limit: 10, timeout: 30000 };
     let url = null;
     let tsv = false;
+    let count = false;
+    let filter = null;
+    let filterRegex = null;
 
     for (let i = 0; i < args.length; i++) {
         const arg = args[i];
@@ -30,6 +36,25 @@ function parseArgs(argv) {
             process.exit(0);
         } else if (arg === '--tsv') {
             tsv = true;
+        } else if (arg === '--count') {
+            count = true;
+        } else if (arg === '--filter') {
+            if (++i >= args.length) {
+                process.stderr.write(`Error: --filter requires a value\n`);
+                process.exit(1);
+            }
+            filter = args[i];
+        } else if (arg === '--filter-regex') {
+            if (++i >= args.length) {
+                process.stderr.write(`Error: --filter-regex requires a value\n`);
+                process.exit(1);
+            }
+            try {
+                filterRegex = new RegExp(args[i]);
+            } catch (e) {
+                process.stderr.write(`Error: --filter-regex invalid regular expression: ${e.message}\n`);
+                process.exit(1);
+            }
         } else if (arg === '--delay') {
             if (++i >= args.length) {
                 process.stderr.write(`Error: --delay requires a value\n`);
@@ -80,33 +105,48 @@ function parseArgs(argv) {
         process.exit(1);
     }
 
-    return { url, opts, tsv };
+    return { url, opts, tsv, count, filter, filterRegex };
 }
 
 (async () => {
-    const { url, opts, tsv } = parseArgs(process.argv);
+    const { url, opts, tsv, count, filter, filterRegex } = parseArgs(process.argv);
 
     const red   = process.stderr.isTTY ? '\x1b[31m' : '';
     const reset = process.stderr.isTTY ? '\x1b[0m'  : '';
 
-    if (tsv) {
+    if (tsv && !count) {
         process.stdout.write('loc\tlastmod\tchangefreq\tpriority\n');
     }
 
     let hasError = false;
+    let filteredCount = 0;
+
+    const hasFilter = filter !== null || filterRegex !== null;
+
+    // onEntry is only skipped when count mode has no filter (result.length is sufficient).
+    const needsOnEntry = !count || hasFilter;
+
     const parser = new SitemapXMLParser(url, {
         ...opts,
-        onEntry: (entry) => {
+        onEntry: needsOnEntry ? (entry) => {
+            const loc = entry.loc ?? '';
+            if (filter !== null && !loc.includes(filter)) return;
+            if (filterRegex !== null && !filterRegex.test(loc)) return;
+
+            if (count) {
+                filteredCount++;
+                return;
+            }
+
             if (tsv) {
-                const loc        = entry.loc?.[0]        ?? '';
-                const lastmod    = entry.lastmod?.[0]    ?? '';
-                const changefreq = entry.changefreq?.[0] ?? '';
-                const priority   = entry.priority?.[0]   ?? '';
+                const lastmod    = entry.lastmod    ?? '';
+                const changefreq = entry.changefreq ?? '';
+                const priority   = entry.priority   ?? '';
                 process.stdout.write(`${loc}\t${lastmod}\t${changefreq}\t${priority}\n`);
             } else {
-                process.stdout.write(entry.loc[0] + '\n');
+                process.stdout.write(loc + '\n');
             }
-        },
+        } : null,
         onError: (failedUrl, err) => {
             hasError = true;
             const msg = err.message.replace(/\r?\n/g, ' ').trim();
@@ -114,6 +154,7 @@ function parseArgs(argv) {
         },
     });
 
-    await parser.fetch();
+    const result = await parser.fetch();
+    if (count) process.stdout.write((hasFilter ? filteredCount : result.length) + '\n');
     if (hasError) process.exit(1);
 })();

package/index.d.ts

@@ -1,8 +1,8 @@
 export interface SitemapEntry {
-    loc: string[];
-    lastmod?: string[];
-    changefreq?: string[];
-    priority?: string[];
+    loc: string;
+    lastmod?: string;
+    changefreq?: string;
+    priority?: string;
 }
 
 export interface SitemapOptions {

package/lib/sitemap.js

@@ -10,13 +10,13 @@ const { URL } = require('url');
 class SitemapXMLParser {
     constructor(url, options = {}) {
         this.siteMapUrl = url;
-        this.delayTime = options.delay ?? 3000;
-        this.limit = options.limit ?? 5;
+        this.delayTime = options.delay ?? 1000;
+        this.limit = options.limit ?? 10;
         this.timeout = options.timeout ?? 30000;
         this.onError = options.onError || null;
         this.onEntry = options.onEntry || null;
         this.urlArray = [];
-        this.parser = new xml2js.Parser();
+        this.parser = new xml2js.Parser({ explicitArray: false });
     }
 
     async fetch() {
@@ -35,7 +35,8 @@ class SitemapXMLParser {
      */
     async getURLFromXML(xml) {
         if (xml.sitemapindex && xml.sitemapindex.sitemap) {
-            const urls = xml.sitemapindex.sitemap.map(s => s.loc?.[0]).filter(Boolean);
+            const sitemapList = [].concat(xml.sitemapindex.sitemap);
+            const urls = sitemapList.map(s => s.loc).filter(Boolean);
 
             for (let i = 0; i < urls.length; i += this.limit) {
                 const chunk = urls.slice(i, i + this.limit);
@@ -55,8 +56,9 @@ class SitemapXMLParser {
         }
 
         if (xml.urlset && xml.urlset.url) {
-            for (const entry of xml.urlset.url) {
-                if (entry && entry.loc?.[0]) {
+            const urlList = [].concat(xml.urlset.url);
+            for (const entry of urlList) {
+                if (entry && entry.loc) {
                     this.urlArray.push(entry);
                     if (this.onEntry) this.onEntry(entry);
                 }
@@ -120,10 +122,11 @@ class SitemapXMLParser {
                     return;
                 }
                 const chunks = [];
+                const contentEncoding = res.headers['content-encoding'];
                 res.on('data', chunk => chunks.push(chunk));
                 res.on('end', () => {
                     const buf = Buffer.concat(chunks);
-                    if (ext === '.gz') {
+                    if (ext === '.gz' || contentEncoding === 'gzip') {
                         zlib.gunzip(buf, (err, result) => {
                             if (err) {
                                 failOnce(originalUrl, err);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sitemap-xml-parser",
-  "version": "1.1.0",
+  "version": "1.3.0",
   "description": "Parses sitemap XML files and returns all listed URLs. Supports sitemap index files and gzip (.gz) compression.",
   "main": "index.js",
   "types": "index.d.ts",
@@ -19,7 +19,7 @@
   "scripts": {
     "test": "node test/test.js"
   },
-  "keywords": ["sitemap", "xml", "parse", "gzip", "sitemap-index", "cli"],
+  "keywords": ["sitemap", "xml", "parse", "gzip", "sitemap-index", "cli", "tsv"],
   "author": "shinkawax",
   "license": "MIT",
   "repository": {