sitemap-xml-parser 1.0.0 → 1.2.0

package/README.md

@@ -8,15 +8,63 @@ Parses sitemap XML files and returns all listed URLs. Supports sitemap index fil
 npm install sitemap-xml-parser
 ```
 
+## CLI
+
+Run without installing via `npx`:
+
+```sh
+npx sitemap-xml-parser <url> [options]
+```
+
+Or, after installing globally (`npm install -g sitemap-xml-parser`):
+
+```sh
+sitemap-xml-parser <url> [options]
+```
+
+Fetched URLs are printed to stdout, one per line. Errors are printed to stderr. See [Options](#options) for available flags.
+
+## Examples
+
+```sh
+# Print all URLs
+npx sitemap-xml-parser https://example.com/sitemap.xml
+
+# Count URLs
+npx sitemap-xml-parser https://example.com/sitemap.xml --count
+
+# Filter by substring
+npx sitemap-xml-parser https://example.com/sitemap.xml --filter "blog"
+
+# Filter and count
+npx sitemap-xml-parser https://example.com/sitemap.xml --filter "blog" --count
+
+# Output as TSV
+npx sitemap-xml-parser https://example.com/sitemap.xml --tsv > urls.tsv
+
+# Save URLs to a file, errors to a log
+npx sitemap-xml-parser https://example.com/sitemap.xml > urls.txt 2> errors.log
+```
+
+## Options
+
+| Option      | Type       | Default | Description                                                                 |
+|-------------|------------|---------|-----------------------------------------------------------------------------|
+| `delay`     | `number`   | `1000`  | Milliseconds to wait between batches when following a sitemap index. Default is 1000 to avoid overloading the target server; set to `0` to disable. CLI: `--delay`   |
+| `limit`     | `number`   | `10`    | Number of child sitemaps to fetch concurrently per batch. CLI: `--limit`              |
+| `timeout`   | `number`   | `30000` | Milliseconds before a request is aborted. CLI: `--timeout`                            |
+| `onError`   | `function` | —       | Called as `onError(url, error)` when a URL fails. The URL is skipped regardless. **Library only.** |
+| `onEntry`   | `function` | —       | Called as `onEntry(entry)` each time a URL entry is parsed. `entry` has the same shape as the objects returned by `fetch()`. **Library only.** |
+| `filter`    | `string`   | —       | Only output URLs whose `loc` contains the given string (substring match). Can be combined with `--count` or `--tsv`. **CLI only.** |
+| `tsv`       | —          | —       | Output results as tab-separated values. Prints a header row (`loc`, `lastmod`, `changefreq`, `priority`) followed by one row per entry. Missing fields are output as empty strings. **CLI only.** |
+| `count`     | —          | —       | Print only the total number of URLs instead of listing them. **CLI only.** |
+
 ## Usage
 
 ```js
 const SitemapXMLParser = require('sitemap-xml-parser');
 
-const parser = new SitemapXMLParser('https://example.com/sitemap.xml', {
-    delay: 3000,
-    limit: 5,
-});
+const parser = new SitemapXMLParser('https://example.com/sitemap.xml');
 
 (async () => {
     const urls = await parser.fetch();
@@ -38,17 +86,6 @@ const parser = new SitemapXMLParser('https://example.com/sitemap.xml', {
 });
 ```
 
-## Options
-
-| Option      | Type       | Default | Description                                                                 |
-|-------------|------------|---------|-----------------------------------------------------------------------------|
-| `delay`     | `number`   | `3000`  | Milliseconds to wait between batches when following a sitemap index. CLI: `--delay`   |
-| `limit`     | `number`   | `5`     | Number of child sitemaps to fetch concurrently per batch. CLI: `--limit`              |
-| `timeout`   | `number`   | `30000` | Milliseconds before a request is aborted. CLI: `--timeout`                            |
-| `onError`   | `function` | —       | Called as `onError(url, error)` when a URL fails. The URL is skipped regardless. **API only.** |
-| `--help`    | —          | —       | Prints usage information and exits. **CLI only.**                           |
-| `--timeout` | —          | —       | Same as the `timeout` option above, in milliseconds. **CLI only.**          |
-
 ## Return value
 
 `fetch()` resolves to an array of URL entry objects. Each object reflects the fields present in the sitemap:
@@ -65,40 +102,10 @@ const parser = new SitemapXMLParser('https://example.com/sitemap.xml', {
 ]
 ```
 
-Fields other than `loc` (`lastmod`, `changefreq`, `priority`, etc.) are included only when present in the source XML.
-
-## CLI
-
-Run without installing via `npx`:
-
-```sh
-npx sitemap-xml-parser <url> [options]
-```
-
-Or, after installing globally (`npm install -g sitemap-xml-parser`):
+All field values are arrays (xml2js convention). Use `entry.loc[0]` to get the URL string, `entry.lastmod?.[0]` for optional fields, and so on.
 
-```sh
-sitemap-xml-parser <url> [options]
-```
-
-Fetched URLs are printed to stdout, one per line. Errors are printed to stderr. See [Options](#options) for available flags.
-
-### Examples
-
-```sh
-# Print all URLs
-npx sitemap-xml-parser https://example.com/sitemap.xml
-
-# No delay, higher concurrency
-npx sitemap-xml-parser https://example.com/sitemap.xml --delay 0 --limit 10
-
-# Save URLs to a file, errors to a log
-npx sitemap-xml-parser https://example.com/sitemap.xml > urls.txt 2> errors.log
-
-# Custom timeout
-npx sitemap-xml-parser https://example.com/sitemap.xml --timeout 10000
-```
+Fields other than `loc` (`lastmod`, `changefreq`, `priority`, etc.) are included only when present in the source XML.
 
 ## Limitations
 
-- **HTTP redirects are not followed.** Responses with status codes 301, 302, or other 3xx are treated as errors. If your sitemap URL redirects, use the final destination URL directly.
+- **HTTP redirects are followed up to 5 times.** Status codes 301, 302, 303, 307, and 308 are handled automatically by following the `Location` header (relative URLs are resolved against the current URL). If the redirect chain exceeds 5 hops, an error is raised via `onError`.

package/bin/cli.js

@@ -8,9 +8,12 @@ function printUsage() {
         'Usage: sitemap-xml-parser <url> [options]',
         '',
         'Options:',
-        '  --delay <ms>    Delay between batches in milliseconds (default: 3000)',
-        '  --limit <n>     Concurrent fetches per batch (default: 5)',
+        '  --delay <ms>    Delay between batches in milliseconds (default: 1000)',
+        '  --limit <n>     Concurrent fetches per batch (default: 10)',
         '  --timeout <ms>  Request timeout in milliseconds (default: 30000)',
+        '  --filter <str>  Only output URLs that contain <str>',
+        '  --tsv           Output as tab-separated values with a header row',
+        '  --count         Print only the total number of URLs',
         '  --help          Show this help message',
         '',
     ].join('\n'));
@@ -18,14 +21,27 @@ function printUsage() {
 
 function parseArgs(argv) {
     const args = argv.slice(2);
-    const opts = { delay: 3000, limit: 5, timeout: 30000 };
+    const opts = { delay: 1000, limit: 10, timeout: 30000 };
     let url = null;
+    let tsv = false;
+    let count = false;
+    let filter = null;
 
     for (let i = 0; i < args.length; i++) {
         const arg = args[i];
         if (arg === '--help' || arg === '-h') {
             printUsage();
             process.exit(0);
+        } else if (arg === '--tsv') {
+            tsv = true;
+        } else if (arg === '--count') {
+            count = true;
+        } else if (arg === '--filter') {
+            if (++i >= args.length) {
+                process.stderr.write(`Error: --filter requires a value\n`);
+                process.exit(1);
+            }
+            filter = args[i];
         } else if (arg === '--delay') {
             if (++i >= args.length) {
                 process.stderr.write(`Error: --delay requires a value\n`);
@@ -76,24 +92,53 @@ function parseArgs(argv) {
         process.exit(1);
     }
 
-    return { url, opts };
+    return { url, opts, tsv, count, filter };
 }
 
 (async () => {
-    const { url, opts } = parseArgs(process.argv);
+    const { url, opts, tsv, count, filter } = parseArgs(process.argv);
+
+    const red   = process.stderr.isTTY ? '\x1b[31m' : '';
+    const reset = process.stderr.isTTY ? '\x1b[0m'  : '';
+
+    if (tsv && !count) {
+        process.stdout.write('loc\tlastmod\tchangefreq\tpriority\n');
+    }
 
     let hasError = false;
+    let filteredCount = 0;
+
+    // onEntry is only skipped when count mode has no filter (result.length is sufficient).
+    const needsOnEntry = !count || filter !== null;
+
     const parser = new SitemapXMLParser(url, {
         ...opts,
+        onEntry: needsOnEntry ? (entry) => {
+            const loc = entry.loc?.[0] ?? '';
+            if (filter !== null && !loc.includes(filter)) return;
+
+            if (count) {
+                filteredCount++;
+                return;
+            }
+
+            if (tsv) {
+                const lastmod    = entry.lastmod?.[0]    ?? '';
+                const changefreq = entry.changefreq?.[0] ?? '';
+                const priority   = entry.priority?.[0]   ?? '';
+                process.stdout.write(`${loc}\t${lastmod}\t${changefreq}\t${priority}\n`);
+            } else {
+                process.stdout.write(loc + '\n');
+            }
+        } : null,
         onError: (failedUrl, err) => {
             hasError = true;
-            process.stderr.write(`Error: ${failedUrl} — ${err.message}\n`);
+            const msg = err.message.replace(/\r?\n/g, ' ').trim();
+            process.stderr.write(`${red}Error: ${failedUrl} — ${msg}${reset}\n`);
         },
     });
 
-    const entries = await parser.fetch();
-    for (const entry of entries) {
-        process.stdout.write(entry.loc[0] + '\n');
-    }
+    const result = await parser.fetch();
+    if (count) process.stdout.write((filter !== null ? filteredCount : result.length) + '\n');
     if (hasError) process.exit(1);
 })();

package/index.d.ts

@@ -0,0 +1,19 @@
+export interface SitemapEntry {
+    loc: string[];
+    lastmod?: string[];
+    changefreq?: string[];
+    priority?: string[];
+}
+
+export interface SitemapOptions {
+    delay?: number;
+    limit?: number;
+    timeout?: number;
+    onError?: (url: string, error: Error) => void;
+    onEntry?: (entry: SitemapEntry) => void;
+}
+
+export default class SitemapXMLParser {
+    constructor(url: string, options?: SitemapOptions);
+    fetch(): Promise<SitemapEntry[]>;
+}

package/lib/sitemap.js

@@ -10,10 +10,11 @@ const { URL } = require('url');
 class SitemapXMLParser {
     constructor(url, options = {}) {
         this.siteMapUrl = url;
-        this.delayTime = options.delay ?? 3000;
-        this.limit = options.limit ?? 5;
+        this.delayTime = options.delay ?? 1000;
+        this.limit = options.limit ?? 10;
         this.timeout = options.timeout ?? 30000;
         this.onError = options.onError || null;
+        this.onEntry = options.onEntry || null;
         this.urlArray = [];
         this.parser = new xml2js.Parser();
     }
@@ -57,6 +58,7 @@ class SitemapXMLParser {
             for (const entry of xml.urlset.url) {
                 if (entry && entry.loc?.[0]) {
                     this.urlArray.push(entry);
+                    if (this.onEntry) this.onEntry(entry);
                 }
             }
         }
@@ -64,50 +66,80 @@ class SitemapXMLParser {
 
     /**
      * Fetch body from URL using http/https.
+     * Follows redirects (301/302/303/307/308) up to 5 times.
      * Decompresses gzip automatically when the URL ends with .gz.
      * Returns null and calls onError on failure.
      */
     getBodyFromURL(url) {
+        return this._fetchWithRedirect(url, url, 0);
+    }
+
+    _fetchWithRedirect(originalUrl, currentUrl, redirectCount) {
         return new Promise((resolve) => {
+            let settled = false;
+            const failOnce = (url, err) => {
+                if (settled) return;
+                settled = true;
+                this._handleError(url, err);
+                resolve(null);
+            };
+
             let parsedUrl;
             try {
-                parsedUrl = new URL(url);
+                parsedUrl = new URL(currentUrl);
             } catch (err) {
-                this._handleError(url, err);
-                resolve(null);
+                failOnce(originalUrl, err);
                 return;
             }
 
             const ext = path.extname(parsedUrl.pathname);
             const transport = parsedUrl.protocol === 'https:' ? https : http;
 
-            const req = transport.get(url, (res) => {
+            const req = transport.get(currentUrl, (res) => {
+                const REDIRECT_CODES = [301, 302, 303, 307, 308];
+                if (REDIRECT_CODES.includes(res.statusCode)) {
+                    res.resume();
+                    const location = res.headers['location'];
+                    if (!location) {
+                        failOnce(originalUrl, new Error(`HTTP ${res.statusCode} with no Location header`));
+                        return;
+                    }
+                    if (redirectCount >= 5) {
+                        failOnce(originalUrl, new Error('Too many redirects (max 5)'));
+                        return;
+                    }
+                    settled = true;
+                    const nextUrl = new URL(location, currentUrl).href;
+                    resolve(this._fetchWithRedirect(originalUrl, nextUrl, redirectCount + 1));
+                    return;
+                }
+
                 if (res.statusCode < 200 || res.statusCode >= 300) {
                     res.resume();
-                    this._handleError(url, new Error(`HTTP ${res.statusCode}`));
-                    resolve(null);
+                    failOnce(originalUrl, new Error(`HTTP ${res.statusCode}`));
                     return;
                 }
                 const chunks = [];
+                const contentEncoding = res.headers['content-encoding'];
                 res.on('data', chunk => chunks.push(chunk));
                 res.on('end', () => {
                     const buf = Buffer.concat(chunks);
-                    if (ext === '.gz') {
+                    if (ext === '.gz' || contentEncoding === 'gzip') {
                         zlib.gunzip(buf, (err, result) => {
                             if (err) {
-                                this._handleError(url, err);
-                                resolve(null);
+                                failOnce(originalUrl, err);
                             } else {
+                                settled = true;
                                 resolve(result.toString());
                             }
                         });
                     } else {
+                        settled = true;
                         resolve(buf.toString());
                     }
                 });
                 res.on('error', (err) => {
-                    this._handleError(url, err);
-                    resolve(null);
+                    failOnce(originalUrl, err);
                 });
             });
 
@@ -116,8 +148,7 @@ class SitemapXMLParser {
             });
 
             req.on('error', (err) => {
-                this._handleError(url, err);
-                resolve(null);
+                failOnce(originalUrl, err);
             });
         });
     }

package/package.json

@@ -1,23 +1,25 @@
 {
   "name": "sitemap-xml-parser",
-  "version": "1.0.0",
+  "version": "1.2.0",
   "description": "Parses sitemap XML files and returns all listed URLs. Supports sitemap index files and gzip (.gz) compression.",
   "main": "index.js",
+  "types": "index.d.ts",
   "bin": {
     "sitemap-xml-parser": "bin/cli.js"
   },
   "files": [
     "index.js",
+    "index.d.ts",
     "lib",
     "bin"
   ],
   "engines": {
-    "node": ">=18"
+    "node": ">=20"
   },
   "scripts": {
     "test": "node test/test.js"
   },
-  "keywords": ["sitemap", "xml", "parse", "gzip", "sitemap-index", "cli"],
+  "keywords": ["sitemap", "xml", "parse", "gzip", "sitemap-index", "cli", "tsv"],
   "author": "shinkawax",
   "license": "MIT",
   "repository": {