sitemap-xml-parser 1.0.0 → 1.1.0

package/README.md

@@ -13,10 +13,7 @@ npm install sitemap-xml-parser
 ```js
 const SitemapXMLParser = require('sitemap-xml-parser');
 
-const parser = new SitemapXMLParser('https://example.com/sitemap.xml', {
-    delay: 3000,
-    limit: 5,
-});
+const parser = new SitemapXMLParser('https://example.com/sitemap.xml');
 
 (async () => {
     const urls = await parser.fetch();
@@ -42,12 +39,12 @@ const parser = new SitemapXMLParser('https://example.com/sitemap.xml', {
 
 | Option      | Type       | Default | Description                                                                 |
 |-------------|------------|---------|-----------------------------------------------------------------------------|
-| `delay`     | `number`   | `3000`  | Milliseconds to wait between batches when following a sitemap index. CLI: `--delay`   |
+| `delay`     | `number`   | `3000`  | Milliseconds to wait between batches when following a sitemap index. Default is 3000 to avoid overloading the target server; set to `0` to disable. CLI: `--delay`   |
 | `limit`     | `number`   | `5`     | Number of child sitemaps to fetch concurrently per batch. CLI: `--limit`              |
 | `timeout`   | `number`   | `30000` | Milliseconds before a request is aborted. CLI: `--timeout`                            |
-| `onError`   | `function` | —       | Called as `onError(url, error)` when a URL fails. The URL is skipped regardless. **API only.** |
-| `--help`    | —          | —       | Prints usage information and exits. **CLI only.**                           |
-| `--timeout` | —          | —       | Same as the `timeout` option above, in milliseconds. **CLI only.**          |
+| `onError`   | `function` | —       | Called as `onError(url, error)` when a URL fails. The URL is skipped regardless. **Library only.** |
+| `onEntry`   | `function` | —       | Called as `onEntry(entry)` each time a URL entry is parsed. `entry` has the same shape as the objects returned by `fetch()`. **Library only.** |
+| `tsv`     | —          | —       | Output results as tab-separated values. Prints a header row (`loc`, `lastmod`, `changefreq`, `priority`) followed by one row per entry. Missing fields are output as empty strings. **CLI only.** |
 
 ## Return value
 
@@ -65,6 +62,8 @@ const parser = new SitemapXMLParser('https://example.com/sitemap.xml', {
 ]
 ```
 
+All field values are arrays (xml2js convention). Use `entry.loc[0]` to get the URL string, `entry.lastmod?.[0]` for optional fields, and so on.
+
 Fields other than `loc` (`lastmod`, `changefreq`, `priority`, etc.) are included only when present in the source XML.
 
 ## CLI
@@ -97,8 +96,14 @@ npx sitemap-xml-parser https://example.com/sitemap.xml > urls.txt 2> errors.log
 
 # Custom timeout
 npx sitemap-xml-parser https://example.com/sitemap.xml --timeout 10000
+
+# Output as TSV (includes lastmod, changefreq, priority)
+npx sitemap-xml-parser https://example.com/sitemap.xml --tsv
+
+# Save TSV to a file
+npx sitemap-xml-parser https://example.com/sitemap.xml --tsv > urls.tsv
 ```
 
 ## Limitations
 
-- **HTTP redirects are not followed.** Responses with status codes 301, 302, or other 3xx are treated as errors. If your sitemap URL redirects, use the final destination URL directly.
+- **HTTP redirects are followed up to 5 times.** Status codes 301, 302, 303, 307, and 308 are handled automatically by following the `Location` header (relative URLs are resolved against the current URL). If the redirect chain exceeds 5 hops, an error is raised via `onError`.

package/bin/cli.js

@@ -11,6 +11,7 @@ function printUsage() {
         '  --delay <ms>    Delay between batches in milliseconds (default: 3000)',
         '  --limit <n>     Concurrent fetches per batch (default: 5)',
         '  --timeout <ms>  Request timeout in milliseconds (default: 30000)',
+        '  --tsv           Output as tab-separated values with a header row',
         '  --help          Show this help message',
         '',
     ].join('\n'));
@@ -20,12 +21,15 @@ function parseArgs(argv) {
     const args = argv.slice(2);
     const opts = { delay: 3000, limit: 5, timeout: 30000 };
     let url = null;
+    let tsv = false;
 
     for (let i = 0; i < args.length; i++) {
         const arg = args[i];
         if (arg === '--help' || arg === '-h') {
             printUsage();
             process.exit(0);
+        } else if (arg === '--tsv') {
+            tsv = true;
         } else if (arg === '--delay') {
             if (++i >= args.length) {
                 process.stderr.write(`Error: --delay requires a value\n`);
@@ -76,24 +80,40 @@ function parseArgs(argv) {
         process.exit(1);
     }
 
-    return { url, opts };
+    return { url, opts, tsv };
 }
 
 (async () => {
-    const { url, opts } = parseArgs(process.argv);
+    const { url, opts, tsv } = parseArgs(process.argv);
+
+    const red   = process.stderr.isTTY ? '\x1b[31m' : '';
+    const reset = process.stderr.isTTY ? '\x1b[0m'  : '';
+
+    if (tsv) {
+        process.stdout.write('loc\tlastmod\tchangefreq\tpriority\n');
+    }
 
     let hasError = false;
     const parser = new SitemapXMLParser(url, {
         ...opts,
+        onEntry: (entry) => {
+            if (tsv) {
+                const loc        = entry.loc?.[0]        ?? '';
+                const lastmod    = entry.lastmod?.[0]    ?? '';
+                const changefreq = entry.changefreq?.[0] ?? '';
+                const priority   = entry.priority?.[0]   ?? '';
+                process.stdout.write(`${loc}\t${lastmod}\t${changefreq}\t${priority}\n`);
+            } else {
+                process.stdout.write(entry.loc[0] + '\n');
+            }
+        },
         onError: (failedUrl, err) => {
             hasError = true;
-            process.stderr.write(`Error: ${failedUrl} — ${err.message}\n`);
+            const msg = err.message.replace(/\r?\n/g, ' ').trim();
+            process.stderr.write(`${red}Error: ${failedUrl} — ${msg}${reset}\n`);
         },
     });
 
-    const entries = await parser.fetch();
-    for (const entry of entries) {
-        process.stdout.write(entry.loc[0] + '\n');
-    }
+    await parser.fetch();
     if (hasError) process.exit(1);
 })();

package/index.d.ts

@@ -0,0 +1,19 @@
+export interface SitemapEntry {
+    loc: string[];
+    lastmod?: string[];
+    changefreq?: string[];
+    priority?: string[];
+}
+
+export interface SitemapOptions {
+    delay?: number;
+    limit?: number;
+    timeout?: number;
+    onError?: (url: string, error: Error) => void;
+    onEntry?: (entry: SitemapEntry) => void;
+}
+
+export default class SitemapXMLParser {
+    constructor(url: string, options?: SitemapOptions);
+    fetch(): Promise<SitemapEntry[]>;
+}

package/lib/sitemap.js

@@ -14,6 +14,7 @@ class SitemapXMLParser {
         this.limit = options.limit ?? 5;
         this.timeout = options.timeout ?? 30000;
         this.onError = options.onError || null;
+        this.onEntry = options.onEntry || null;
         this.urlArray = [];
         this.parser = new xml2js.Parser();
     }
@@ -57,6 +58,7 @@ class SitemapXMLParser {
             for (const entry of xml.urlset.url) {
                 if (entry && entry.loc?.[0]) {
                     this.urlArray.push(entry);
+                    if (this.onEntry) this.onEntry(entry);
                 }
             }
         }
@@ -64,28 +66,57 @@ class SitemapXMLParser {
 
     /**
      * Fetch body from URL using http/https.
+     * Follows redirects (301/302/303/307/308) up to 5 times.
      * Decompresses gzip automatically when the URL ends with .gz.
      * Returns null and calls onError on failure.
      */
     getBodyFromURL(url) {
+        return this._fetchWithRedirect(url, url, 0);
+    }
+
+    _fetchWithRedirect(originalUrl, currentUrl, redirectCount) {
         return new Promise((resolve) => {
+            let settled = false;
+            const failOnce = (url, err) => {
+                if (settled) return;
+                settled = true;
+                this._handleError(url, err);
+                resolve(null);
+            };
+
             let parsedUrl;
             try {
-                parsedUrl = new URL(url);
+                parsedUrl = new URL(currentUrl);
             } catch (err) {
-                this._handleError(url, err);
-                resolve(null);
+                failOnce(originalUrl, err);
                 return;
             }
 
             const ext = path.extname(parsedUrl.pathname);
             const transport = parsedUrl.protocol === 'https:' ? https : http;
 
-            const req = transport.get(url, (res) => {
+            const req = transport.get(currentUrl, (res) => {
+                const REDIRECT_CODES = [301, 302, 303, 307, 308];
+                if (REDIRECT_CODES.includes(res.statusCode)) {
+                    res.resume();
+                    const location = res.headers['location'];
+                    if (!location) {
+                        failOnce(originalUrl, new Error(`HTTP ${res.statusCode} with no Location header`));
+                        return;
+                    }
+                    if (redirectCount >= 5) {
+                        failOnce(originalUrl, new Error('Too many redirects (max 5)'));
+                        return;
+                    }
+                    settled = true;
+                    const nextUrl = new URL(location, currentUrl).href;
+                    resolve(this._fetchWithRedirect(originalUrl, nextUrl, redirectCount + 1));
+                    return;
+                }
+
                 if (res.statusCode < 200 || res.statusCode >= 300) {
                     res.resume();
-                    this._handleError(url, new Error(`HTTP ${res.statusCode}`));
-                    resolve(null);
+                    failOnce(originalUrl, new Error(`HTTP ${res.statusCode}`));
                     return;
                 }
                 const chunks = [];
@@ -95,19 +126,19 @@ class SitemapXMLParser {
                     if (ext === '.gz') {
                         zlib.gunzip(buf, (err, result) => {
                             if (err) {
-                                this._handleError(url, err);
-                                resolve(null);
+                                failOnce(originalUrl, err);
                             } else {
+                                settled = true;
                                 resolve(result.toString());
                             }
                         });
                     } else {
+                        settled = true;
                         resolve(buf.toString());
                     }
                 });
                 res.on('error', (err) => {
-                    this._handleError(url, err);
-                    resolve(null);
+                    failOnce(originalUrl, err);
                 });
             });
 
@@ -116,8 +147,7 @@ class SitemapXMLParser {
             });
 
             req.on('error', (err) => {
-                this._handleError(url, err);
-                resolve(null);
+                failOnce(originalUrl, err);
             });
         });
     }

package/package.json

@@ -1,18 +1,20 @@
 {
   "name": "sitemap-xml-parser",
-  "version": "1.0.0",
+  "version": "1.1.0",
   "description": "Parses sitemap XML files and returns all listed URLs. Supports sitemap index files and gzip (.gz) compression.",
   "main": "index.js",
+  "types": "index.d.ts",
   "bin": {
     "sitemap-xml-parser": "bin/cli.js"
   },
   "files": [
     "index.js",
+    "index.d.ts",
     "lib",
     "bin"
   ],
   "engines": {
-    "node": ">=18"
+    "node": ">=20"
   },
   "scripts": {
     "test": "node test/test.js"