postal-code-scraper 1.0.2

package/build/test/src/utils/logger.js

@@ -0,0 +1,87 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Logger = void 0;
+class Logger {
+    static configure(config) {
+        if (config.level)
+            this.logLevel = config.level;
+        if (config.colors !== undefined)
+            this.useColors = config.colors;
+        if (config.prefix)
+            this.prefix = config.prefix;
+        if (config.logger)
+            this.instance = config.logger;
+    }
+    static getInstance() {
+        return this.instance || new Logger();
+    }
+    static debug(message, ...args) {
+        this.log("debug", message, args);
+    }
+    static info(message, ...args) {
+        this.log("info", message, args);
+    }
+    static warn(message, ...args) {
+        this.log("warn", message, args);
+    }
+    static error(message, ...args) {
+        this.log("error", message, args);
+    }
+    static shouldLog(level) {
+        if (this.logLevel === "silent")
+            return false;
+        const levels = ["error", "warn", "info", "debug"];
+        return levels.indexOf(level) <= levels.indexOf(this.logLevel);
+    }
+    static log(level, message, args) {
+        if (!this.shouldLog(level))
+            return;
+        const logger = this.getInstance();
+        const formatted = this.formatMessage(level, message);
+        logger[level](formatted, ...args);
+    }
+    static formatMessage(level, message) {
+        const timestamp = new Date().toISOString();
+        const levelColor = this.getLevelColor(level);
+        const messageColor = this.useColors ? "\x1b[37m" : "";
+        return [
+            this.useColors ? "\x1b[90m" : "",
+            `${this.prefix} `,
+            `${timestamp} `,
+            levelColor,
+            `[${level.toUpperCase()}]`,
+            this.useColors ? "\x1b[0m" : "",
+            messageColor,
+            ` ${message}`,
+            this.useColors ? "\x1b[0m" : "",
+        ].join("");
+    }
+    static getLevelColor(level) {
+        if (!this.useColors)
+            return "";
+        return {
+            error: "\x1b[31m", // Red
+            warn: "\x1b[33m", // Yellow
+            info: "\x1b[36m", // Cyan
+            debug: "\x1b[35m", // Magenta
+        }[level];
+    }
+    // Instance methods to implement LoggerInterface
+    debug(message, ...args) {
+        console.debug(message, ...args);
+    }
+    info(message, ...args) {
+        console.log(message, ...args);
+    }
+    warn(message, ...args) {
+        console.warn(message, ...args);
+    }
+    error(message, ...args) {
+        console.error(message, ...args);
+    }
+}
+exports.Logger = Logger;
+Logger.logLevel = "info";
+Logger.useColors = true;
+Logger.prefix = "[POSTAL-CODE-SCRAPER]";
+//# sourceMappingURL=logger.js.map

package/build/test/src/utils/logger.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"logger.js","sourceRoot":"","sources":["../../../../src/utils/logger.ts"],"names":[],"mappings":";;;AAUA,MAAa,MAAM;IAMlB,MAAM,CAAC,SAAS,CAAC,MAAyF;QACzG,IAAI,MAAM,CAAC,KAAK;YAAE,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC;QAC/C,IAAI,MAAM,CAAC,MAAM,KAAK,SAAS;YAAE,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC;QAChE,IAAI,MAAM,CAAC,MAAM;YAAE,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;QAC/C,IAAI,MAAM,CAAC,MAAM;YAAE,IAAI,CAAC,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC;IAClD,CAAC;IAED,MAAM,CAAC,WAAW;QACjB,OAAO,IAAI,CAAC,QAAQ,IAAI,IAAI,MAAM,EAAE,CAAC;IACtC,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,OAAe,EAAE,GAAG,IAAW;QAC3C,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC;IAClC,CAAC;IAED,MAAM,CAAC,IAAI,CAAC,OAAe,EAAE,GAAG,IAAW;QAC1C,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC;IACjC,CAAC;IAED,MAAM,CAAC,IAAI,CAAC,OAAe,EAAE,GAAG,IAAW;QAC1C,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC;IACjC,CAAC;IAED,MAAM,CAAC,KAAK,CAAC,OAAe,EAAE,GAAG,IAAW;QAC3C,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC;IAClC,CAAC;IAEO,MAAM,CAAC,SAAS,CAAC,KAAe;QACvC,IAAI,IAAI,CAAC,QAAQ,KAAK,QAAQ;YAAE,OAAO,KAAK,CAAC;QAE7C,MAAM,MAAM,GAAgB,CAAC,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;QAC/D,OAAO,MAAM,CAAC,OAAO,CAAC,KAAkB,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,QAAqB,CAAC,CAAC;IACzF,CAAC;IAEO,MAAM,CAAC,GAAG,CAAC,KAAgB,EAAE,OAAe,EAAE,IAAW;QAChE,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC;YAAE,OAAO;QAEnC,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;QAClC,MAAM,SAAS,GAAG,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QAErD,MAAM,CAAC,KAAK,CAAC,CAAC,SAAS,EAAE,GAAG,IAAI,CAAC,CAAC;IACnC,CAAC;IAEO,MAAM,CAAC,aAAa,CAAC,KAAgB,EAAE,OAAe;QAC7D,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QAC3C,MAAM,UAAU,GAAG,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;QAC7C,MAAM,YAAY,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,CAAC;QAEtD,OAAO;YACN,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE;YAChC,GAAG,IAAI,CAAC,MAAM,GAAG;YACjB,GAAG,SAAS,GAAG;YACf,UAAU;YACV,IAAI,KAAK,CAAC,WAAW,EAAE,GAAG;YAC1B,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE;YAC/B,YAAY;YACZ,IAAI,OAAO,EAAE;YACb,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE;SAC/B,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACZ,CAAC;IAEO,MAAM,CAAC,aAAa,CAAC,KAAgB;QAC5C,IAAI,CAAC,IAAI,CAAC,SAAS;YAAE,OAAO,EAAE,CAAC;QAE/B,OAAO;YACN,KAAK,EAAE,UAAU,EAAE,MAAM;YACzB,IAAI,EAAE,UAAU,EAAE,SAAS;YAC3B,IAAI,EAAE,UAAU,EAAE,OAAO;YACzB,KAAK,EAAE,UAAU,EAAE,UAAU;SAC7B,CAAC,KAAK,CAAC,CAAC;IACV,CAAC;IAED,gDAAgD;IAChD,KAAK,CAAC,OAAe,EAAE,GAAG,IAAW;QACpC,OAAO,CAAC,KAAK,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,CAAC;IACjC,CAAC;IAED,IAAI,CAAC,OAAe,EAAE,GAAG,IAAW;QACnC,OAAO,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,CAAC;IAC/B,CAAC;IAED,IAAI,CAAC,OAAe,EAAE,GAAG,IAAW;QACnC,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,CAAC;IAChC,CAAC;IAED,KAAK,CAAC,OAAe,EAAE,GAAG,IAAW;QACpC,OAAO,CAAC,KAAK,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,CAAC;IACjC,CAAC;;AA7FF,wBA8FC;AA7Fe,eAAQ,GAAa,MAAM,CAAC;AAC5B,gBAAS,GAAY,IAAI,CAAC;AAC1B,aAAM,GAAW,uBAAuB,CAAC"}

package/build/test/tests/postal-code-scraper.test.js

@@ -0,0 +1,14 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const chai_1 = require("chai");
+const index_1 = __importDefault(require("../src/index"));
+describe("Postal Code Scraper", () => {
+    it("should do something", () => {
+        const result = index_1.default.scrapeCountry("romania");
+        (0, chai_1.expect)(result).to.be.ok;
+    });
+});
+//# sourceMappingURL=postal-code-scraper.test.js.map

package/build/test/tests/postal-code-scraper.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"postal-code-scraper.test.js","sourceRoot":"","sources":["../../../tests/postal-code-scraper.test.ts"],"names":[],"mappings":";;;;;AAAA,+BAA8B;AAC9B,yDAA6C;AAE7C,QAAQ,CAAC,qBAAqB,EAAE,GAAG,EAAE;IACpC,EAAE,CAAC,qBAAqB,EAAE,GAAG,EAAE;QAC9B,MAAM,MAAM,GAAG,eAAiB,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC;QAC1D,IAAA,aAAM,EAAC,MAAM,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC;IACzB,CAAC,CAAC,CAAC;AACJ,CAAC,CAAC,CAAC"}

package/dist/index.d.ts

@@ -0,0 +1,3 @@
+export * from "./types";
+export { PostalCodeScraper } from "./scraper/scrapers";
+export { default } from "./scraper/scrapers";

package/dist/index.js

@@ -0,0 +1,25 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = exports.PostalCodeScraper = void 0;
+__exportStar(require("./types"), exports);
+var scrapers_1 = require("./scraper/scrapers");
+Object.defineProperty(exports, "PostalCodeScraper", { enumerable: true, get: function () { return scrapers_1.PostalCodeScraper; } });
+var scrapers_2 = require("./scraper/scrapers");
+Object.defineProperty(exports, "default", { enumerable: true, get: function () { return __importDefault(scrapers_2).default; } });

package/dist/scraper/fetchers.d.ts

@@ -0,0 +1,9 @@
+import { Browser } from "puppeteer";
+import { ScraperConfig } from "../types";
+export declare class Fetcher {
+    private browser;
+    private config;
+    constructor(browser: Browser, config: ScraperConfig);
+    fetchHtml(url: string): Promise<string>;
+    fetchWithRetry(url: string, retries?: number): Promise<string>;
+}

package/dist/scraper/fetchers.js

@@ -0,0 +1,48 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Fetcher = void 0;
+class Fetcher {
+    constructor(browser, config) {
+        this.browser = browser;
+        this.config = config;
+    }
+    fetchHtml(url) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const page = yield this.browser.newPage();
+            try {
+                page.setDefaultNavigationTimeout(60000);
+                yield page.goto(url, { waitUntil: "domcontentloaded" });
+                return yield page.content();
+            }
+            finally {
+                yield page.close();
+            }
+        });
+    }
+    fetchWithRetry(url_1) {
+        return __awaiter(this, arguments, void 0, function* (url, retries = this.config.maxRetries || 5) {
+            var _a;
+            try {
+                return yield this.fetchHtml(url);
+            }
+            catch (error) {
+                (_a = this.config.logger) === null || _a === void 0 ? void 0 : _a.warn(`Retrying (${this.config.maxRetries - retries + 1}) for: ${url}`);
+                if (retries > 0) {
+                    yield new Promise((resolve) => setTimeout(resolve, Math.random() * 7000 + 5000));
+                    return this.fetchWithRetry(url, retries - 1);
+                }
+                throw new Error(`Failed to fetch: ${url} after ${this.config.maxRetries} attempts`);
+            }
+        });
+    }
+}
+exports.Fetcher = Fetcher;

package/dist/scraper/parsers.d.ts

@@ -0,0 +1,7 @@
+import { Region, ScraperConfig } from "../types";
+export declare class Parser {
+    static parseRegions($: cheerio.Root, config: ScraperConfig): Region[];
+    static parsePostalCodes($: cheerio.Root, config: ScraperConfig): Record<string, string[]>;
+    static parseCountries($: cheerio.Root, config: ScraperConfig): Region[];
+    static parseCountryByName($: cheerio.Root, config: ScraperConfig, name: string): Region | null;
+}

package/dist/scraper/parsers.js

@@ -0,0 +1,62 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Parser = void 0;
+class Parser {
+    static parseRegions($, config) {
+        return $("h2:contains('Regions')")
+            .next(".regions")
+            .find("a")
+            .map((_index, element) => {
+            const path = $(element).attr("href");
+            const prettyName = $(element).text().trim();
+            if (!path || !prettyName)
+                return null;
+            return {
+                name: path.split("/").filter(Boolean).pop(),
+                prettyName,
+                path,
+            };
+        })
+            .get()
+            .filter(Boolean);
+    }
+    static parsePostalCodes($, config) {
+        const codes = {};
+        $(".codes .container").each((_i, element) => {
+            const place = $(element).find(".place").text().trim();
+            const codesList = $(element)
+                .find(".code span")
+                .map((_j, el) => $(el).text().trim())
+                .get();
+            if (place) {
+                const key = config.usePrettyName ? place : place.toLowerCase().replace(/\s+/g, "-");
+                codes[key] = codesList;
+            }
+        });
+        return codes;
+    }
+    static parseCountries($, config) {
+        return $(".regions div a")
+            .map((_i, element) => {
+            const path = $(element).attr("href");
+            return path ? { name: path.replace(/\//g, ""), prettyName: $(element).text().trim(), path } : null;
+        })
+            .get()
+            .filter(Boolean);
+    }
+    static parseCountryByName($, config, name) {
+        const countryElement = $(`.regions div a`).filter((_, el) => { var _a; return ((_a = $(el).attr("href")) === null || _a === void 0 ? void 0 : _a.replace(/\//g, "")) === name.toLowerCase().trim(); });
+        if (!countryElement.length)
+            return null;
+        const path = countryElement.attr("href");
+        const prettyName = countryElement.text().trim();
+        return path && prettyName
+            ? {
+                name: path.replace(/\//g, ""),
+                prettyName,
+                path,
+            }
+            : null;
+    }
+}
+exports.Parser = Parser;

package/dist/scraper/queue.d.ts

@@ -0,0 +1,12 @@
+import { Region, ScraperConfig, RegionData } from "../types";
+import { Fetcher } from "./fetchers";
+export declare class ProcessingQueue {
+    private fetcher;
+    private config;
+    private queue;
+    private visitedUrls;
+    private limit;
+    constructor(fetcher: Fetcher, config: ScraperConfig);
+    process(startRegion: Region, data: RegionData): Promise<void>;
+    private processItem;
+}

package/dist/scraper/queue.js

@@ -0,0 +1,67 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ProcessingQueue = void 0;
+const cheerio_1 = require("cheerio");
+const p_limit_1 = __importDefault(require("p-limit"));
+const parsers_1 = require("./parsers");
+const env_config_1 = require("../utils/env-config");
+class ProcessingQueue {
+    constructor(fetcher, config) {
+        this.fetcher = fetcher;
+        this.config = config;
+        this.queue = [];
+        this.visitedUrls = new Set();
+        this.limit = (0, p_limit_1.default)(config.concurrency || 15);
+    }
+    process(startRegion, data) {
+        return __awaiter(this, void 0, void 0, function* () {
+            this.queue.push({ region: startRegion, currData: data });
+            while (this.queue.length > 0) {
+                const tasks = this.queue.map((item) => this.limit(() => this.processItem(item)));
+                this.queue = [];
+                yield Promise.all(tasks);
+            }
+        });
+    }
+    processItem(item) {
+        return __awaiter(this, void 0, void 0, function* () {
+            var _a, _b;
+            const url = `${(0, env_config_1.getBaseUrl)()}${item.region.path}`;
+            if (this.visitedUrls.has(url))
+                return;
+            this.visitedUrls.add(url);
+            (_a = this.config.logger) === null || _a === void 0 ? void 0 : _a.info(`Fetching: ${url}`);
+            try {
+                const html = yield this.fetcher.fetchWithRetry(url);
+                const $ = (0, cheerio_1.load)(html);
+                const regions = parsers_1.Parser.parseRegions($, this.config);
+                regions.forEach((region) => {
+                    const key = this.config.usePrettyName ? region.prettyName : region.name;
+                    item.currData[key] = {};
+                    this.queue.push({
+                        region,
+                        currData: item.currData[key],
+                    });
+                });
+                const codes = parsers_1.Parser.parsePostalCodes($, this.config);
+                Object.assign(item.currData, codes);
+            }
+            catch (error) {
+                (_b = this.config.logger) === null || _b === void 0 ? void 0 : _b.error(`Error processing ${url}:`, error);
+            }
+        });
+    }
+}
+exports.ProcessingQueue = ProcessingQueue;

package/dist/scraper/scrapers.d.ts

@@ -0,0 +1,19 @@
+import { ScraperConfig } from "../types";
+export declare class PostalCodeScraper {
+    private config;
+    private browser;
+    private queue;
+    private fetcher;
+    constructor(config?: ScraperConfig);
+    scrapeCountry(countryName: string): Promise<null | undefined>;
+    scrapeCountries(): Promise<null | undefined>;
+    private initBrowser;
+    private getCountryDetails;
+    private getCountriesDetails;
+    private generatePostalCodeLookup;
+    private buildLookup;
+    private saveData;
+    private cleanup;
+}
+declare const _default: PostalCodeScraper;
+export default _default;

package/dist/scraper/scrapers.js

@@ -0,0 +1,149 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PostalCodeScraper = void 0;
+const path_1 = __importDefault(require("path"));
+const puppeteer_1 = __importDefault(require("puppeteer"));
+const queue_1 = require("./queue");
+const fetchers_1 = require("./fetchers");
+const id_generator_1 = require("../utils/id-generator");
+const fs_1 = require("fs");
+const cheerio_1 = require("cheerio");
+const parsers_1 = require("./parsers");
+const env_config_1 = require("../utils/env-config");
+const string_utils_1 = require("../utils/string-utils");
+const logger_1 = require("../utils/logger");
+class PostalCodeScraper {
+    constructor(config = {}) {
+        this.config = config;
+        this.config = Object.assign({ concurrency: 15, maxRetries: 5, headless: true, directory: "src/data", logger: logger_1.Logger, usePrettyName: false }, config);
+    }
+    scrapeCountry(countryName) {
+        return __awaiter(this, void 0, void 0, function* () {
+            var _a;
+            yield this.initBrowser();
+            try {
+                const country = yield this.getCountryDetails(countryName);
+                if (!country) {
+                    (_a = this.config.logger) === null || _a === void 0 ? void 0 : _a.warn(`Country not found: ${countryName}`);
+                    return null;
+                }
+                const data = {};
+                yield this.queue.process(country, data);
+                this.saveData(data, `${country.name}-postal-codes.json`, this.config.directory);
+                const postalCodeLookup = this.generatePostalCodeLookup(data);
+                this.saveData(postalCodeLookup, `${country.name}-lookup.json`, this.config.directory);
+            }
+            finally {
+                yield this.cleanup();
+            }
+        });
+    }
+    scrapeCountries() {
+        return __awaiter(this, void 0, void 0, function* () {
+            var _a, _b;
+            yield this.initBrowser();
+            try {
+                const countries = yield this.getCountriesDetails();
+                if (countries.length === 0) {
+                    (_a = this.config.logger) === null || _a === void 0 ? void 0 : _a.warn("No countries found.");
+                    return null;
+                }
+                for (const country of countries) {
+                    const key = this.config.usePrettyName ? country.prettyName : country.name;
+                    const countryData = {};
+                    (_b = this.config.logger) === null || _b === void 0 ? void 0 : _b.info(`Processing country: ${key}`);
+                    yield this.queue.process(country, countryData);
+                    this.saveData(countryData, `${key}-postal-codes.json`, this.config.directory);
+                    const postalCodeLookup = this.generatePostalCodeLookup(countryData);
+                    this.saveData(postalCodeLookup, `${key}-lookup.json`, this.config.directory);
+                }
+            }
+            finally {
+                yield this.cleanup();
+            }
+        });
+    }
+    initBrowser() {
+        return __awaiter(this, void 0, void 0, function* () {
+            this.browser = yield puppeteer_1.default.launch({ headless: this.config.headless });
+            this.fetcher = new fetchers_1.Fetcher(this.browser, this.config);
+            this.queue = new queue_1.ProcessingQueue(this.fetcher, this.config);
+        });
+    }
+    getCountryDetails(name) {
+        return __awaiter(this, void 0, void 0, function* () {
+            var _a;
+            try {
+                const html = yield this.fetcher.fetchWithRetry((0, env_config_1.getBaseUrl)());
+                return parsers_1.Parser.parseCountryByName((0, cheerio_1.load)(html), this.config, name);
+            }
+            catch (error) {
+                (_a = this.config.logger) === null || _a === void 0 ? void 0 : _a.error(`Error fetching country details: ${name}`, error);
+                return null;
+            }
+        });
+    }
+    getCountriesDetails() {
+        return __awaiter(this, void 0, void 0, function* () {
+            var _a;
+            try {
+                const html = yield this.fetcher.fetchWithRetry((0, env_config_1.getBaseUrl)());
+                return parsers_1.Parser.parseCountries((0, cheerio_1.load)(html), this.config);
+            }
+            catch (error) {
+                (_a = this.config.logger) === null || _a === void 0 ? void 0 : _a.error("Error fetching countries details", error);
+                return [];
+            }
+        });
+    }
+    generatePostalCodeLookup(data) {
+        return this.buildLookup(data, (0, id_generator_1.createRegionIdGenerator)());
+    }
+    buildLookup(regionObj, idGenerator, acc = [], result = { postalCodeMap: {}, regions: {} }) {
+        if (Array.isArray(regionObj)) {
+            for (const item of regionObj) {
+                const id = idGenerator(acc);
+                result.postalCodeMap[item] = id;
+                result.regions[id] = [...acc];
+            }
+        }
+        else if (typeof regionObj === "object" && regionObj !== null) {
+            for (const [regionKey, regionValue] of Object.entries(regionObj)) {
+                this.buildLookup(regionValue, idGenerator, [...acc, regionKey], result);
+            }
+        }
+        return result;
+    }
+    saveData(data, fileName, directory = "src/data") {
+        var _a, _b;
+        try {
+            (0, fs_1.mkdirSync)(directory, { recursive: true });
+            const filePath = path_1.default.join(directory, (0, string_utils_1.normalizeString)(fileName));
+            (0, fs_1.writeFileSync)(filePath, JSON.stringify(data, null, 2), { flag: "w" });
+            (_a = this.config.logger) === null || _a === void 0 ? void 0 : _a.info(`Saved data to ${filePath}`);
+        }
+        catch (error) {
+            (_b = this.config.logger) === null || _b === void 0 ? void 0 : _b.error(`Error saving data to ${fileName}`, error);
+        }
+    }
+    cleanup() {
+        return __awaiter(this, void 0, void 0, function* () {
+            var _a;
+            yield ((_a = this.browser) === null || _a === void 0 ? void 0 : _a.close());
+        });
+    }
+}
+exports.PostalCodeScraper = PostalCodeScraper;
+exports.default = new PostalCodeScraper();

package/dist/types.d.ts

@@ -0,0 +1,32 @@
+export type Region = {
+    path: string;
+    name: string;
+    prettyName: string;
+};
+export type ScraperConfig = {
+    usePrettyName?: boolean;
+    directory?: string;
+    concurrency?: number;
+    maxRetries?: number;
+    headless?: boolean;
+    logger?: any;
+};
+export type ProcessingQueueItem = {
+    region: Region;
+    currData: RegionData;
+};
+export interface LookupData {
+    postalCodeMap: {
+        [postalCode: string]: string;
+    };
+    regions: {
+        [code: string]: string[];
+    };
+}
+export interface PostalCodeData {
+    rawData: RegionData;
+    postalCodeLookup: LookupData;
+}
+export interface RegionData {
+    [key: string]: RegionData | string[];
+}

package/dist/types.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/utils/env-config.d.ts

@@ -0,0 +1 @@
+export declare const getBaseUrl: () => string;

package/dist/utils/env-config.js

@@ -0,0 +1,7 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getBaseUrl = void 0;
+const getBaseUrl = () => {
+    return "https://worldpostalcode.com";
+};
+exports.getBaseUrl = getBaseUrl;

package/dist/utils/id-generator.d.ts

@@ -0,0 +1,4 @@
+export interface RegionIdGenerator {
+    (regions: string[]): string;
+}
+export declare const createRegionIdGenerator: () => RegionIdGenerator;

package/dist/utils/id-generator.js

@@ -0,0 +1,26 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createRegionIdGenerator = void 0;
+const createRegionIdGenerator = () => {
+    const regionRegistry = new Map();
+    const counterMap = new Map();
+    return (regions) => {
+        const normalized = regions.map((region) => region
+            .trim()
+            .toLowerCase()
+            .normalize("NFD")
+            .replace(/[\u0300-\u036f]/g, "")
+            .replace(/\s+/g, "_"));
+        const compositeKey = normalized.join("|");
+        if (regionRegistry.has(compositeKey)) {
+            return regionRegistry.get(compositeKey);
+        }
+        const baseName = normalized[normalized.length - 1];
+        const count = (counterMap.get(baseName) || 0) + 1;
+        counterMap.set(baseName, count);
+        const newId = `${baseName}_${count}`;
+        regionRegistry.set(compositeKey, newId);
+        return newId;
+    };
+};
+exports.createRegionIdGenerator = createRegionIdGenerator;

package/dist/utils/logger.d.ts

@@ -0,0 +1,33 @@
+export type LogMethod = "error" | "warn" | "info" | "debug";
+export type LogLevel = LogMethod | "silent";
+export interface LoggerInterface {
+    debug(message: string, ...args: any[]): void;
+    info(message: string, ...args: any[]): void;
+    warn(message: string, ...args: any[]): void;
+    error(message: string, ...args: any[]): void;
+}
+export declare class Logger implements LoggerInterface {
+    private static logLevel;
+    private static useColors;
+    private static prefix;
+    private static instance;
+    static configure(config: {
+        level?: LogLevel;
+        colors?: boolean;
+        prefix?: string;
+        logger?: LoggerInterface;
+    }): void;
+    static getInstance(): LoggerInterface;
+    static debug(message: string, ...args: any[]): void;
+    static info(message: string, ...args: any[]): void;
+    static warn(message: string, ...args: any[]): void;
+    static error(message: string, ...args: any[]): void;
+    private static shouldLog;
+    private static log;
+    private static formatMessage;
+    private static getLevelColor;
+    debug(message: string, ...args: any[]): void;
+    info(message: string, ...args: any[]): void;
+    warn(message: string, ...args: any[]): void;
+    error(message: string, ...args: any[]): void;
+}