postal-code-scraper 1.0.2 → 1.0.4

package/README.md

@@ -4,15 +4,15 @@
 
 **Postal Code Scraper** is an automated web scraper designed to extract postal code data from countries worldwide. It efficiently fetches postal codes and organizes them into structured JSON files for easy use in applications.
 
-This library uses **Puppeteer** for web scraping and **Cheerio** for HTML parsing, ensuring accurate and efficient data extraction.
+This library uses **Puppeteer** for web scraping, **Cheerio** for HTML parsing, **p-limit** for controlling concurrency, ensuring accurate and efficient data extraction.
 
 ## 🚀 Features
 
-- Scrape **postal codes** from any country
-- Scrape **all countries** in one go
+- Scrape **postal codes** for one country or all countries
 - Save results as **JSON** files for easy integration
-- Configurable settings (concurrency, retries, headless mode, etc.)  <- read more below
-- Structured **postal code lookup** generation
+- **Region-structured** output (country → region1 → region2 → region3 → ... → postal codes)
+- **Postal-code lookup** output (postal code → region path)
+- Configurable options (concurrency, retries, headless mode, output directory, logging, etc.)
 - **Fully asynchronous** for optimized performance
 
 ## 📦 Installation
@@ -45,50 +45,65 @@ import { PostalCodeScraper } from "postal-code-scraper";
 const { PostalCodeScraper } = require("postal-code-scraper");
 ```
 
-### 2️⃣ **Scrape a Single Country**
+### 2️⃣ **Instantiate Scraper**
 
 ```javascript
-async function scrapeSingleCountry() {
-    await PostalCodeScraper.scrapeCountry("Canada");
-}
+const scraper = new PostalCodeScraper();
+```
+
+### 3️⃣ **Scrape a Single Country**
+
+```javascript
+import { PostalCodeScraper } from "postal-code-scraper";
 
-scrapeSingleCountry();
+const scraper = new PostalCodeScraper();
+
+await scraper.scrapeCountry("Romania");
 ```
 
 📌 **Output Files (saved in **``**):**
 
-- `Canada-postal-codes.json`
-- `Canada-lookup.json`
+- `romania-postal-codes.json`
+- `romania-lookup.json`
 
-### 3️⃣ **Scrape All Countries**
+### 4️⃣ **Scrape All Countries**
 
 ```javascript
-async function scrapeAllCountries() {
-    await PostalCodeScraper.scrapeCountries();
-}
+import { PostalCodeScraper } from "postal-code-scraper";
 
-scrapeAllCountries();
+const scraper = new PostalCodeScraper();
+
+await scraper.scrapeCountries();
 ```
 
 📌 This will fetch postal codes for **every available country**.
 
-### 4️⃣ **Customize Scraper Configuration**
+### 5️⃣ **Customize Scraper Configuration**
+
+#### 🛠 Configuration Options
+
+| Option          | Type            | Default                          | Description                                                  |
+| --------------- | --------------- | -------------------------------- | ------------------------------------------------------------ |
+| `directory`     | `string`        | `src/data`                       | The directory to save data                                   |
+| `concurrency`   | `number`        | `15`                             | Maximum concurrent requests to process                       |
+| `maxRetries`    | `number`        | `5`                              | Number of retries for failed requests                        |
+| `headless`      | `boolean`       | `true`                           | Run Puppeteer in headless mode                               |
+| `usePrettyName` | `boolean`       | `false`                          | Use country pretty names instead of default names            |
+| `logger`        | `object` `null` | `Logger` (custom implementation) | Handles event logging, can be set to null to disable logging |
 
 ```javascript
+import { PostalCodeScraper } from "postal-code-scraper";
+
 const customScraper = new PostalCodeScraper({
-    concurrency: 10,  // Limit concurrent requests
-    maxRetries: 3,    // Max retries per request (if a request fails -> so we don't lose data)
-    headless: false,  // Run Puppeteer in visible mode
-    usePrettyName: true, // Store data using country pretty names
-    logger: console  // Enable console logging (default is own implemented) 
-    directory: 'src/data'  // Choose the folder where you want to save the data
+	concurrency: 10, // Limit concurrent requests
+	maxRetries: 3, // Max retries per request
+	headless: false, // Run Puppeteer in visible mode
+	usePrettyName: true, // Store data using country pretty names
+	logger: console, // Enable console logging (set to null to disable)
+	directory: "src/data", // Output directory
 });
 
-async function run() {
-    await customScraper.scrapeCountry("Germany");
-}
-
-run();
+await customScraper.scrapeCountry("Germany");
 ```
 
 ## 📁 Output Data Format
@@ -117,41 +132,21 @@ run();
 
 ```json
 {
-  "postalCodeMap": {
-    "337563": "tamasesti_2",
-    "337564": "valea_4",
-    "400001": "cluj-napoca_1",
-    "400002": "cluj-napoca_1",
-    "400003": "cluj-napoca_1",
-  },
-  "regions": {
-    "cluj-napoca_1": [
-      "cluj",
-      "cluj-napoca"
-    ],
-    "tamasesti_2": [
-      "hunedoara",
-      "tamasesti"
-    ],
-    "valea_4": [
-      "hunedoara",
-      "valea"
-    ],
-  }
+	"postalCodeMap": {
+		"337563": "tamasesti_2",
+		"337564": "valea_4",
+		"400001": "cluj-napoca_1",
+		"400002": "cluj-napoca_1",
+		"400003": "cluj-napoca_1"
+	},
+	"regions": {
+		"cluj-napoca_1": ["cluj", "cluj-napoca"],
+		"tamasesti_2": ["hunedoara", "tamasesti"],
+		"valea_4": ["hunedoara", "valea"]
+	}
 }
 ```
 
-## 🛠 Configuration Options
-
-| Option          | Type                          | Default                          | Description                                                                                    |
-| --------------- | ----------------------------- | -------------------------------- | ---------------------------------------------------------------------------------------------- |
-| `directory`     | `string`                      | `src/data`                       | The directory to save data                                                                     |
-| `concurrency`   | `number`                      | `15`                             | Maximum concurrent requests to process                                                         |
-| `maxRetries`    | `number`                      | `5`                              | Number of retries for failed requests                                                          |
-| `headless`      | `boolean`                     | `true`                           | Run Puppeteer in headless mode                                                                 |
-| `usePrettyName` | `boolean`                     | `false`                          | Use country pretty names instead of default names                                              |
-| `logger`        | `object` `null`               | `Logger` (custom implementation) | Handles event logging, can be set to null to disable logging                                   |
-
 ## ❓ FAQs
 
 ### **1. Where are the postal code files stored?**
@@ -191,4 +186,3 @@ Contributions are welcome! Feel free to submit a pull request or open an issue.
 ## 📜 License
 
 MIT License © 2024
-

package/dist/index.cjs

@@ -0,0 +1,392 @@
+"use strict";
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/index.ts
+var index_exports = {};
+__export(index_exports, {
+  PostalCodeScraper: () => PostalCodeScraper
+});
+module.exports = __toCommonJS(index_exports);
+
+// src/scraper/scrapers.ts
+var import_path = __toESM(require("path"), 1);
+var import_puppeteer = __toESM(require("puppeteer"), 1);
+
+// src/scraper/queue.ts
+var import_cheerio = require("cheerio");
+var import_p_limit = __toESM(require("p-limit"), 1);
+
+// src/scraper/parsers.ts
+var Parser = class {
+  static parseRegions($, config) {
+    return $("h2:contains('Regions')").next(".regions").find("a").map((_index, element) => {
+      const path2 = $(element).attr("href");
+      const prettyName = $(element).text().trim();
+      if (!path2 || !prettyName) return null;
+      return {
+        name: path2.split("/").filter(Boolean).pop(),
+        prettyName,
+        path: path2
+      };
+    }).get().filter(Boolean);
+  }
+  static parsePostalCodes($, config) {
+    const codes = {};
+    $(".codes .container").each((_i, element) => {
+      const place = $(element).find(".place").text().trim();
+      const codesList = $(element).find(".code span").map((_j, el) => $(el).text().trim()).get();
+      if (place) {
+        const key = config.usePrettyName ? place : place.toLowerCase().replace(/\s+/g, "-");
+        codes[key] = codesList;
+      }
+    });
+    return codes;
+  }
+  static parseCountries($, config) {
+    return $(".regions div a").map((_i, element) => {
+      const path2 = $(element).attr("href");
+      return path2 ? { name: path2.replace(/\//g, ""), prettyName: $(element).text().trim(), path: path2 } : null;
+    }).get().filter(Boolean);
+  }
+  static parseCountryByName($, config, name) {
+    const countryElement = $(`.regions div a`).filter((_, el) => $(el).attr("href")?.replace(/\//g, "") === name.toLowerCase().trim());
+    if (!countryElement.length) return null;
+    const path2 = countryElement.attr("href");
+    const prettyName = countryElement.text().trim();
+    return path2 && prettyName ? {
+      name: path2.replace(/\//g, ""),
+      prettyName,
+      path: path2
+    } : null;
+  }
+};
+
+// src/utils/env-config.ts
+var getBaseUrl = () => {
+  return "https://worldpostalcode.com";
+};
+
+// src/scraper/queue.ts
+var ProcessingQueue = class {
+  constructor(fetcher, config) {
+    this.fetcher = fetcher;
+    this.config = config;
+    this.limit = (0, import_p_limit.default)(config.concurrency || 15);
+  }
+  queue = [];
+  visitedUrls = /* @__PURE__ */ new Set();
+  limit;
+  async process(startRegion, data) {
+    this.queue.push({ region: startRegion, currData: data });
+    while (this.queue.length > 0) {
+      const tasks = this.queue.map((item) => this.limit(() => this.processItem(item)));
+      this.queue = [];
+      await Promise.all(tasks);
+    }
+  }
+  async processItem(item) {
+    const url = `${getBaseUrl()}${item.region.path}`;
+    if (this.visitedUrls.has(url)) return;
+    this.visitedUrls.add(url);
+    this.config.logger?.info(`Fetching: ${url}`);
+    try {
+      const html = await this.fetcher.fetchWithRetry(url);
+      const $ = (0, import_cheerio.load)(html);
+      const regions = Parser.parseRegions($, this.config);
+      regions.forEach((region) => {
+        const key = this.config.usePrettyName ? region.prettyName : region.name;
+        item.currData[key] = {};
+        this.queue.push({
+          region,
+          currData: item.currData[key]
+        });
+      });
+      const codes = Parser.parsePostalCodes($, this.config);
+      Object.assign(item.currData, codes);
+    } catch (error) {
+      this.config.logger?.error(`Error processing ${url}:`, error);
+    }
+  }
+};
+
+// src/scraper/fetchers.ts
+var Fetcher = class {
+  constructor(browser, config) {
+    this.browser = browser;
+    this.config = config;
+  }
+  async fetchHtml(url) {
+    const page = await this.browser.newPage();
+    try {
+      page.setDefaultNavigationTimeout(6e4);
+      await page.goto(url, { waitUntil: "domcontentloaded" });
+      return await page.content();
+    } finally {
+      await page.close();
+    }
+  }
+  async fetchWithRetry(url, retries = this.config.maxRetries || 5) {
+    try {
+      return await this.fetchHtml(url);
+    } catch (error) {
+      this.config.logger?.warn(`Retrying (${this.config.maxRetries - retries + 1}) for: ${url}`);
+      if (retries > 0) {
+        await new Promise((resolve) => setTimeout(resolve, Math.random() * 7e3 + 5e3));
+        return this.fetchWithRetry(url, retries - 1);
+      }
+      throw new Error(`Failed to fetch: ${url} after ${this.config.maxRetries} attempts`);
+    }
+  }
+};
+
+// src/utils/id-generator.ts
+var createRegionIdGenerator = () => {
+  const regionRegistry = /* @__PURE__ */ new Map();
+  const counterMap = /* @__PURE__ */ new Map();
+  return (regions) => {
+    const normalized = regions.map(
+      (region) => region.trim().toLowerCase().normalize("NFD").replace(/[\u0300-\u036f]/g, "").replace(/\s+/g, "_")
+    );
+    const compositeKey = normalized.join("|");
+    if (regionRegistry.has(compositeKey)) {
+      return regionRegistry.get(compositeKey);
+    }
+    const baseName = normalized[normalized.length - 1];
+    const count = (counterMap.get(baseName) || 0) + 1;
+    counterMap.set(baseName, count);
+    const newId = `${baseName}_${count}`;
+    regionRegistry.set(compositeKey, newId);
+    return newId;
+  };
+};
+
+// src/scraper/scrapers.ts
+var import_fs = require("fs");
+var import_cheerio2 = require("cheerio");
+
+// src/utils/string-utils.ts
+var normalizeString = (str) => {
+  return str.trim().toLowerCase().normalize("NFD").replace(/[\u0300-\u036f]/g, "").replace(/\s+/g, "-").replace(/[^a-z0-9.-]/g, "");
+};
+
+// src/utils/logger.ts
+var Logger = class _Logger {
+  static logLevel = "info";
+  static useColors = true;
+  static prefix = "[POSTAL-CODE-SCRAPER]";
+  static instance;
+  static configure(config) {
+    if (config.level) this.logLevel = config.level;
+    if (config.colors !== void 0) this.useColors = config.colors;
+    if (config.prefix) this.prefix = config.prefix;
+    if (config.logger) this.instance = config.logger;
+  }
+  static getInstance() {
+    return this.instance || new _Logger();
+  }
+  static debug(message, ...args) {
+    this.log("debug", message, args);
+  }
+  static info(message, ...args) {
+    this.log("info", message, args);
+  }
+  static warn(message, ...args) {
+    this.log("warn", message, args);
+  }
+  static error(message, ...args) {
+    this.log("error", message, args);
+  }
+  static shouldLog(level) {
+    if (this.logLevel === "silent") return false;
+    const levels = ["error", "warn", "info", "debug"];
+    return levels.indexOf(level) <= levels.indexOf(this.logLevel);
+  }
+  static log(level, message, args) {
+    if (!this.shouldLog(level)) return;
+    const logger = this.getInstance();
+    const formatted = this.formatMessage(level, message);
+    logger[level](formatted, ...args);
+  }
+  static formatMessage(level, message) {
+    const timestamp = (/* @__PURE__ */ new Date()).toISOString();
+    const levelColor = this.getLevelColor(level);
+    const messageColor = this.useColors ? "\x1B[37m" : "";
+    return [
+      this.useColors ? "\x1B[90m" : "",
+      `${this.prefix} `,
+      `${timestamp} `,
+      levelColor,
+      `[${level.toUpperCase()}]`,
+      this.useColors ? "\x1B[0m" : "",
+      messageColor,
+      ` ${message}`,
+      this.useColors ? "\x1B[0m" : ""
+    ].join("");
+  }
+  static getLevelColor(level) {
+    if (!this.useColors) return "";
+    return {
+      error: "\x1B[31m",
+      // Red
+      warn: "\x1B[33m",
+      // Yellow
+      info: "\x1B[36m",
+      // Cyan
+      debug: "\x1B[35m"
+      // Magenta
+    }[level];
+  }
+  // Instance methods to implement LoggerInterface
+  debug(message, ...args) {
+    console.debug(message, ...args);
+  }
+  info(message, ...args) {
+    console.log(message, ...args);
+  }
+  warn(message, ...args) {
+    console.warn(message, ...args);
+  }
+  error(message, ...args) {
+    console.error(message, ...args);
+  }
+};
+
+// src/scraper/scrapers.ts
+var PostalCodeScraper = class {
+  constructor(config = {}) {
+    this.config = config;
+    this.config = {
+      concurrency: 15,
+      maxRetries: 5,
+      headless: true,
+      directory: "src/data",
+      logger: Logger,
+      usePrettyName: false,
+      ...config
+    };
+  }
+  browser;
+  queue;
+  fetcher;
+  async scrapeCountry(countryName) {
+    await this.initBrowser();
+    try {
+      const country = await this.getCountryDetails(countryName);
+      if (!country) {
+        this.config.logger?.warn(`Country not found: ${countryName}`);
+        return null;
+      }
+      const data = {};
+      await this.queue.process(country, data);
+      this.saveData(data, `${country.name}-postal-codes.json`, this.config.directory);
+      const postalCodeLookup = this.generatePostalCodeLookup(data);
+      this.saveData(postalCodeLookup, `${country.name}-lookup.json`, this.config.directory);
+    } finally {
+      await this.cleanup();
+    }
+  }
+  async scrapeCountries() {
+    await this.initBrowser();
+    try {
+      const countries = await this.getCountriesDetails();
+      if (countries.length === 0) {
+        this.config.logger?.warn("No countries found.");
+        return null;
+      }
+      for (const country of countries) {
+        const key = this.config.usePrettyName ? country.prettyName : country.name;
+        const countryData = {};
+        this.config.logger?.info(`Processing country: ${key}`);
+        await this.queue.process(country, countryData);
+        this.saveData(countryData, `${key}-postal-codes.json`, this.config.directory);
+        const postalCodeLookup = this.generatePostalCodeLookup(countryData);
+        this.saveData(postalCodeLookup, `${key}-lookup.json`, this.config.directory);
+      }
+    } finally {
+      await this.cleanup();
+    }
+  }
+  async initBrowser() {
+    this.browser = await import_puppeteer.default.launch({ headless: this.config.headless });
+    this.fetcher = new Fetcher(this.browser, this.config);
+    this.queue = new ProcessingQueue(this.fetcher, this.config);
+  }
+  async getCountryDetails(name) {
+    try {
+      const html = await this.fetcher.fetchWithRetry(getBaseUrl());
+      return Parser.parseCountryByName((0, import_cheerio2.load)(html), this.config, name);
+    } catch (error) {
+      this.config.logger?.error(`Error fetching country details: ${name}`, error);
+      return null;
+    }
+  }
+  async getCountriesDetails() {
+    try {
+      const html = await this.fetcher.fetchWithRetry(getBaseUrl());
+      return Parser.parseCountries((0, import_cheerio2.load)(html), this.config);
+    } catch (error) {
+      this.config.logger?.error("Error fetching countries details", error);
+      return [];
+    }
+  }
+  generatePostalCodeLookup(data) {
+    return this.buildLookup(data, createRegionIdGenerator());
+  }
+  buildLookup(regionObj, idGenerator, acc = [], result = { postalCodeMap: {}, regions: {} }) {
+    if (Array.isArray(regionObj)) {
+      for (const item of regionObj) {
+        const id = idGenerator(acc);
+        result.postalCodeMap[item] = id;
+        result.regions[id] = [...acc];
+      }
+    } else if (typeof regionObj === "object" && regionObj !== null) {
+      for (const [regionKey, regionValue] of Object.entries(regionObj)) {
+        this.buildLookup(regionValue, idGenerator, [...acc, regionKey], result);
+      }
+    }
+    return result;
+  }
+  saveData(data, fileName, directory = "src/data") {
+    try {
+      (0, import_fs.mkdirSync)(directory, { recursive: true });
+      const filePath = import_path.default.join(directory, normalizeString(fileName));
+      (0, import_fs.writeFileSync)(filePath, JSON.stringify(data, null, 2), { flag: "w" });
+      this.config.logger?.info(`Saved data to ${filePath}`);
+    } catch (error) {
+      this.config.logger?.error(`Error saving data to ${fileName}`, error);
+    }
+  }
+  async cleanup() {
+    await this.browser?.close();
+  }
+};
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  PostalCodeScraper
+});

package/dist/index.d.cts

@@ -0,0 +1,43 @@
+type Region = {
+    path: string;
+    name: string;
+    prettyName: string;
+};
+type ScraperConfig = {
+    usePrettyName?: boolean;
+    directory?: string;
+    concurrency?: number;
+    maxRetries?: number;
+    headless?: boolean;
+    logger?: any;
+};
+interface LookupData {
+    postalCodeMap: {
+        [postalCode: string]: string;
+    };
+    regions: {
+        [code: string]: string[];
+    };
+}
+interface RegionData {
+    [key: string]: RegionData | string[];
+}
+
+declare class PostalCodeScraper {
+    private config;
+    private browser;
+    private queue;
+    private fetcher;
+    constructor(config?: ScraperConfig);
+    scrapeCountry(countryName: string): Promise<null | undefined>;
+    scrapeCountries(): Promise<null | undefined>;
+    private initBrowser;
+    private getCountryDetails;
+    private getCountriesDetails;
+    private generatePostalCodeLookup;
+    private buildLookup;
+    private saveData;
+    private cleanup;
+}
+
+export { type LookupData, PostalCodeScraper, type Region, type RegionData, type ScraperConfig };

package/dist/index.d.ts

@@ -1,3 +1,43 @@
-export * from "./types";
-export { PostalCodeScraper } from "./scraper/scrapers";
-export { default } from "./scraper/scrapers";
+type Region = {
+    path: string;
+    name: string;
+    prettyName: string;
+};
+type ScraperConfig = {
+    usePrettyName?: boolean;
+    directory?: string;
+    concurrency?: number;
+    maxRetries?: number;
+    headless?: boolean;
+    logger?: any;
+};
+interface LookupData {
+    postalCodeMap: {
+        [postalCode: string]: string;
+    };
+    regions: {
+        [code: string]: string[];
+    };
+}
+interface RegionData {
+    [key: string]: RegionData | string[];
+}
+
+declare class PostalCodeScraper {
+    private config;
+    private browser;
+    private queue;
+    private fetcher;
+    constructor(config?: ScraperConfig);
+    scrapeCountry(countryName: string): Promise<null | undefined>;
+    scrapeCountries(): Promise<null | undefined>;
+    private initBrowser;
+    private getCountryDetails;
+    private getCountriesDetails;
+    private generatePostalCodeLookup;
+    private buildLookup;
+    private saveData;
+    private cleanup;
+}
+
+export { type LookupData, PostalCodeScraper, type Region, type RegionData, type ScraperConfig };