postal-code-scraper 1.0.2

package/dist/utils/logger.js

@@ -0,0 +1,86 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.Logger = void 0;
+class Logger {
+    static configure(config) {
+        if (config.level)
+            this.logLevel = config.level;
+        if (config.colors !== undefined)
+            this.useColors = config.colors;
+        if (config.prefix)
+            this.prefix = config.prefix;
+        if (config.logger)
+            this.instance = config.logger;
+    }
+    static getInstance() {
+        return this.instance || new Logger();
+    }
+    static debug(message, ...args) {
+        this.log("debug", message, args);
+    }
+    static info(message, ...args) {
+        this.log("info", message, args);
+    }
+    static warn(message, ...args) {
+        this.log("warn", message, args);
+    }
+    static error(message, ...args) {
+        this.log("error", message, args);
+    }
+    static shouldLog(level) {
+        if (this.logLevel === "silent")
+            return false;
+        const levels = ["error", "warn", "info", "debug"];
+        return levels.indexOf(level) <= levels.indexOf(this.logLevel);
+    }
+    static log(level, message, args) {
+        if (!this.shouldLog(level))
+            return;
+        const logger = this.getInstance();
+        const formatted = this.formatMessage(level, message);
+        logger[level](formatted, ...args);
+    }
+    static formatMessage(level, message) {
+        const timestamp = new Date().toISOString();
+        const levelColor = this.getLevelColor(level);
+        const messageColor = this.useColors ? "\x1b[37m" : "";
+        return [
+            this.useColors ? "\x1b[90m" : "",
+            `${this.prefix} `,
+            `${timestamp} `,
+            levelColor,
+            `[${level.toUpperCase()}]`,
+            this.useColors ? "\x1b[0m" : "",
+            messageColor,
+            ` ${message}`,
+            this.useColors ? "\x1b[0m" : "",
+        ].join("");
+    }
+    static getLevelColor(level) {
+        if (!this.useColors)
+            return "";
+        return {
+            error: "\x1b[31m", // Red
+            warn: "\x1b[33m", // Yellow
+            info: "\x1b[36m", // Cyan
+            debug: "\x1b[35m", // Magenta
+        }[level];
+    }
+    // Instance methods to implement LoggerInterface
+    debug(message, ...args) {
+        console.debug(message, ...args);
+    }
+    info(message, ...args) {
+        console.log(message, ...args);
+    }
+    warn(message, ...args) {
+        console.warn(message, ...args);
+    }
+    error(message, ...args) {
+        console.error(message, ...args);
+    }
+}
+exports.Logger = Logger;
+Logger.logLevel = "info";
+Logger.useColors = true;
+Logger.prefix = "[POSTAL-CODE-SCRAPER]";

package/dist/utils/string-utils.d.ts

@@ -0,0 +1 @@
+export declare const normalizeString: (str: string) => string;

package/dist/utils/string-utils.js

@@ -0,0 +1,13 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.normalizeString = void 0;
+const normalizeString = (str) => {
+    return str
+        .trim()
+        .toLowerCase()
+        .normalize("NFD")
+        .replace(/[\u0300-\u036f]/g, "")
+        .replace(/\s+/g, "-")
+        .replace(/[^a-z0-9.-]/g, "");
+};
+exports.normalizeString = normalizeString;

package/package.json

@@ -0,0 +1,61 @@
+{
+	"name": "postal-code-scraper",
+	"version": "1.0.2",
+	"description": "A tool for scraping country data, including regions and their postal codes",
+	"main": "dist/index.js",
+	"types": "dist/index.d.ts",
+	"scripts": {
+		"start": "tsx src/index.ts",
+		"prebuild": "shx rm -rf dist",
+		"build": "tsc",
+		"pretest": "npm run build",
+		"test": "mocha -r tsx tests/**/*.test.ts",
+		"prepare": "npm run build",
+		"pack": "npm pack",
+		"prepublishOnly": "npm test"
+	},
+	"type": "module",
+	"devDependencies": {
+		"@types/chai": "^5.0.1",
+		"@types/cheerio": "^0.22.35",
+		"@types/mocha": "^10.0.10",
+		"@types/node": "^22.13.9",
+		"@types/p-limit": "^2.1.0",
+		"@types/puppeteer": "^5.4.7",
+		"@types/sinon": "^17.0.4",
+		"chai": "^5.2.0",
+		"mocha": "^11.1.0",
+		"sinon": "^19.0.2",
+		"tslib": "^2.8.1",
+		"tsx": "^3.14.0",
+		"typescript": "^5.8.2"
+	},
+	"dependencies": {
+		"cheerio": "^1.0.0",
+		"p-limit": "^6.2.0",
+		"puppeteer": "^24.2.1"
+	},
+	"repository": {
+		"type": "git",
+		"url": "git+https://github.com/sasiasas/postal-code-scraper.git"
+	},
+	"keywords": [
+		"postal-codes",
+		"scraper",
+		"geodata",
+		"zip-codes",
+		"region",
+		"country",
+		"county",
+		"location",
+		"address"
+	],
+	"author": "Sallai Tamás-Levente",
+	"license": "MIT",
+	"bugs": {
+		"url": "https://github.com/sasiasas/postal-code-scraper/issues"
+	},
+	"engines": {
+		"node": ">=18"
+	}
+}

package/src/index.ts

@@ -0,0 +1,3 @@
+export * from "./types";
+export { PostalCodeScraper } from "./scraper/scrapers";
+export { default } from "./scraper/scrapers";

package/src/scraper/fetchers.ts

@@ -0,0 +1,30 @@
+import { Browser } from "puppeteer";
+import { ScraperConfig } from "../types";
+
+export class Fetcher {
+	constructor(private browser: Browser, private config: ScraperConfig) {}
+
+	async fetchHtml(url: string): Promise<string> {
+		const page = await this.browser.newPage();
+		try {
+			page.setDefaultNavigationTimeout(60000);
+			await page.goto(url, { waitUntil: "domcontentloaded" });
+			return await page.content();
+		} finally {
+			await page.close();
+		}
+	}
+
+	async fetchWithRetry(url: string, retries = this.config.maxRetries || 5): Promise<string> {
+		try {
+			return await this.fetchHtml(url);
+		} catch (error) {
+			this.config.logger?.warn(`Retrying (${this.config.maxRetries! - retries + 1}) for: ${url}`);
+			if (retries > 0) {
+				await new Promise((resolve) => setTimeout(resolve, Math.random() * 7000 + 5000));
+				return this.fetchWithRetry(url, retries - 1);
+			}
+			throw new Error(`Failed to fetch: ${url} after ${this.config.maxRetries} attempts`);
+		}
+	}
+}

package/src/scraper/parsers.ts

@@ -0,0 +1,67 @@
+import { Region, ScraperConfig } from "../types";
+
+export class Parser {
+	static parseRegions($: cheerio.Root, config: ScraperConfig): Region[] {
+		return $("h2:contains('Regions')")
+			.next(".regions")
+			.find("a")
+			.map((_index, element) => {
+				const path = $(element).attr("href");
+				const prettyName = $(element).text().trim();
+				if (!path || !prettyName) return null;
+
+				return {
+					name: path.split("/").filter(Boolean).pop()!,
+					prettyName,
+					path,
+				};
+			})
+			.get()
+			.filter(Boolean) as Region[];
+	}
+
+	static parsePostalCodes($: cheerio.Root, config: ScraperConfig): Record<string, string[]> {
+		const codes: Record<string, string[]> = {};
+
+		$(".codes .container").each((_i, element) => {
+			const place = $(element).find(".place").text().trim();
+			const codesList = $(element)
+				.find(".code span")
+				.map((_j, el) => $(el).text().trim())
+				.get();
+
+			if (place) {
+				const key = config.usePrettyName ? place : place.toLowerCase().replace(/\s+/g, "-");
+				codes[key] = codesList;
+			}
+		});
+
+		return codes;
+	}
+
+	static parseCountries($: cheerio.Root, config: ScraperConfig): Region[] {
+		return $(".regions div a")
+			.map((_i, element) => {
+				const path = $(element).attr("href");
+				return path ? { name: path.replace(/\//g, ""), prettyName: $(element).text().trim(), path } : null;
+			})
+			.get()
+			.filter(Boolean) as Region[];
+	}
+
+	static parseCountryByName($: cheerio.Root, config: ScraperConfig, name: string): Region | null {
+		const countryElement = $(`.regions div a`).filter((_, el) => $(el).attr("href")?.replace(/\//g, "") === name.toLowerCase().trim());
+
+		if (!countryElement.length) return null;
+
+		const path = countryElement.attr("href");
+		const prettyName = countryElement.text().trim();
+		return path && prettyName
+			? {
+					name: path.replace(/\//g, ""),
+					prettyName,
+					path,
+			  }
+			: null;
+	}
+}

package/src/scraper/queue.ts

@@ -0,0 +1,55 @@
+import { load } from "cheerio";
+import { Region, ProcessingQueueItem, ScraperConfig, RegionData } from "../types";
+import { Fetcher } from "./fetchers";
+import pLimit from "p-limit";
+import { Parser } from "./parsers";
+import { getBaseUrl } from "../utils/env-config";
+
+export class ProcessingQueue {
+	private queue: ProcessingQueueItem[] = [];
+	private visitedUrls = new Set<string>();
+	private limit: ReturnType<typeof pLimit>;
+
+	constructor(private fetcher: Fetcher, private config: ScraperConfig) {
+		this.limit = pLimit(config.concurrency || 15);
+	}
+
+	async process(startRegion: Region, data: RegionData): Promise<void> {
+		this.queue.push({ region: startRegion, currData: data });
+
+		while (this.queue.length > 0) {
+			const tasks = this.queue.map((item) => this.limit(() => this.processItem(item)));
+			this.queue = [];
+			await Promise.all(tasks);
+		}
+	}
+
+	private async processItem(item: ProcessingQueueItem): Promise<void> {
+		const url = `${getBaseUrl()}${item.region.path}`;
+
+		if (this.visitedUrls.has(url)) return;
+		this.visitedUrls.add(url);
+
+		this.config.logger?.info(`Fetching: ${url}`);
+
+		try {
+			const html = await this.fetcher.fetchWithRetry(url);
+			const $ = load(html);
+
+			const regions = Parser.parseRegions($, this.config);
+			regions.forEach((region) => {
+				const key = this.config.usePrettyName ? region.prettyName : region.name;
+				item.currData[key] = {};
+				this.queue.push({
+					region,
+					currData: item.currData[key],
+				});
+			});
+
+			const codes = Parser.parsePostalCodes($, this.config);
+			Object.assign(item.currData, codes);
+		} catch (error) {
+			this.config.logger?.error(`Error processing ${url}:`, error);
+		}
+	}
+}

package/src/scraper/scrapers.ts

@@ -0,0 +1,143 @@
+import path from "path";
+import puppeteer, { Browser } from "puppeteer";
+import { ProcessingQueue } from "./queue";
+import { Fetcher } from "./fetchers";
+import { Region, ScraperConfig, LookupData, RegionData } from "../types";
+import { createRegionIdGenerator, RegionIdGenerator } from "../utils/id-generator";
+import { writeFileSync, mkdirSync } from "fs";
+import { load } from "cheerio";
+import { Parser } from "./parsers";
+import { getBaseUrl } from "../utils/env-config";
+import { normalizeString } from "../utils/string-utils";
+import { Logger } from "../utils/logger";
+
+export class PostalCodeScraper {
+	private browser!: Browser;
+	private queue!: ProcessingQueue;
+	private fetcher!: Fetcher;
+
+	constructor(private config: ScraperConfig = {}) {
+		this.config = {
+			concurrency: 15,
+			maxRetries: 5,
+			headless: true,
+			directory: "src/data",
+			logger: Logger,
+			usePrettyName: false,
+			...config,
+		};
+	}
+
+	async scrapeCountry(countryName: string) {
+		await this.initBrowser();
+		try {
+			const country = await this.getCountryDetails(countryName);
+			if (!country) {
+				this.config.logger?.warn(`Country not found: ${countryName}`);
+				return null;
+			}
+
+			const data: RegionData = {};
+			await this.queue.process(country, data);
+
+			this.saveData(data, `${country.name}-postal-codes.json`, this.config.directory!);
+
+			const postalCodeLookup = this.generatePostalCodeLookup(data);
+			this.saveData(postalCodeLookup, `${country.name}-lookup.json`, this.config.directory!);
+		} finally {
+			await this.cleanup();
+		}
+	}
+
+	async scrapeCountries() {
+		await this.initBrowser();
+		try {
+			const countries = await this.getCountriesDetails();
+			if (countries.length === 0) {
+				this.config.logger?.warn("No countries found.");
+				return null;
+			}
+
+			for (const country of countries) {
+				const key = this.config.usePrettyName ? country.prettyName : country.name;
+				const countryData: RegionData = {};
+				this.config.logger?.info(`Processing country: ${key}`);
+
+				await this.queue.process(country, countryData);
+				this.saveData(countryData, `${key}-postal-codes.json`, this.config.directory!);
+
+				const postalCodeLookup = this.generatePostalCodeLookup(countryData);
+				this.saveData(postalCodeLookup, `${key}-lookup.json`, this.config.directory!);
+			}
+		} finally {
+			await this.cleanup();
+		}
+	}
+
+	private async initBrowser() {
+		this.browser = await puppeteer.launch({ headless: this.config.headless });
+		this.fetcher = new Fetcher(this.browser, this.config);
+		this.queue = new ProcessingQueue(this.fetcher, this.config);
+	}
+
+	private async getCountryDetails(name: string): Promise<Region | null> {
+		try {
+			const html = await this.fetcher.fetchWithRetry(getBaseUrl());
+			return Parser.parseCountryByName(load(html), this.config, name);
+		} catch (error) {
+			this.config.logger?.error(`Error fetching country details: ${name}`, error);
+			return null;
+		}
+	}
+
+	private async getCountriesDetails(): Promise<Region[]> {
+		try {
+			const html = await this.fetcher.fetchWithRetry(getBaseUrl());
+			return Parser.parseCountries(load(html), this.config);
+		} catch (error) {
+			this.config.logger?.error("Error fetching countries details", error);
+			return [];
+		}
+	}
+
+	private generatePostalCodeLookup(data: RegionData): LookupData {
+		return this.buildLookup(data, createRegionIdGenerator());
+	}
+
+	private buildLookup(
+		regionObj: RegionData | string[],
+		idGenerator: RegionIdGenerator,
+		acc: string[] = [],
+		result: LookupData = { postalCodeMap: {}, regions: {} }
+	): LookupData {
+		if (Array.isArray(regionObj)) {
+			for (const item of regionObj) {
+				const id = idGenerator(acc);
+				result.postalCodeMap[item] = id;
+				result.regions[id] = [...acc];
+			}
+		} else if (typeof regionObj === "object" && regionObj !== null) {
+			for (const [regionKey, regionValue] of Object.entries(regionObj)) {
+				this.buildLookup(regionValue, idGenerator, [...acc, regionKey], result);
+			}
+		}
+		return result;
+	}
+
+	private saveData(data: any, fileName: string, directory: string = "src/data") {
+		try {
+			mkdirSync(directory, { recursive: true });
+			const filePath = path.join(directory, normalizeString(fileName));
+			writeFileSync(filePath, JSON.stringify(data, null, 2), { flag: "w" });
+			this.config.logger?.info(`Saved data to ${filePath}`);
+		} catch (error) {
+			this.config.logger?.error(`Error saving data to ${fileName}`, error);
+		}
+	}
+
+	private async cleanup() {
+		await this.browser?.close();
+	}
+}
+
+export default new PostalCodeScraper();

package/src/types.ts

@@ -0,0 +1,37 @@
+export type Region = {
+	path: string;
+	name: string;
+	prettyName: string;
+};
+
+export type ScraperConfig = {
+	usePrettyName?: boolean;
+	directory?: string;
+	concurrency?: number;
+	maxRetries?: number;
+	headless?: boolean;
+	logger?: any;
+};
+
+export type ProcessingQueueItem = {
+	region: Region;
+	currData: RegionData;
+};
+
+export interface LookupData {
+	postalCodeMap: {
+		[postalCode: string]: string;
+	};
+	regions: {
+		[code: string]: string[];
+	};
+}
+
+export interface PostalCodeData {
+	rawData: RegionData;
+	postalCodeLookup: LookupData;
+}
+
+export interface RegionData {
+	[key: string]: RegionData | string[];
+}

package/src/utils/env-config.ts

@@ -0,0 +1,3 @@
+export const getBaseUrl = () => {
+	return "https://worldpostalcode.com";
+};

package/src/utils/id-generator.ts

@@ -0,0 +1,35 @@
+export interface RegionIdGenerator {
+	(regions: string[]): string;
+}
+
+export const createRegionIdGenerator = (): RegionIdGenerator => {
+	const regionRegistry = new Map<string, string>();
+	const counterMap = new Map<string, number>();
+
+	return (regions: string[]): string => {
+		const normalized = regions.map((region) =>
+			region
+				.trim()
+				.toLowerCase()
+				.normalize("NFD")
+				.replace(/[\u0300-\u036f]/g, "")
+				.replace(/\s+/g, "_")
+		);
+
+		const compositeKey = normalized.join("|");
+
+		if (regionRegistry.has(compositeKey)) {
+			return regionRegistry.get(compositeKey)!;
+		}
+
+		const baseName = normalized[normalized.length - 1];
+		const count = (counterMap.get(baseName) || 0) + 1;
+		counterMap.set(baseName, count);
+
+		const newId = `${baseName}_${count}`;
+
+		regionRegistry.set(compositeKey, newId);
+
+		return newId;
+	};
+};

package/src/utils/logger.ts

@@ -0,0 +1,105 @@
+export type LogMethod = "error" | "warn" | "info" | "debug";
+export type LogLevel = LogMethod | "silent";
+
+export interface LoggerInterface {
+	debug(message: string, ...args: any[]): void;
+	info(message: string, ...args: any[]): void;
+	warn(message: string, ...args: any[]): void;
+	error(message: string, ...args: any[]): void;
+}
+
+export class Logger implements LoggerInterface {
+	private static logLevel: LogLevel = "info";
+	private static useColors: boolean = true;
+	private static prefix: string = "[POSTAL-CODE-SCRAPER]";
+	private static instance: LoggerInterface;
+
+	static configure(config: { level?: LogLevel; colors?: boolean; prefix?: string; logger?: LoggerInterface }): void {
+		if (config.level) this.logLevel = config.level;
+		if (config.colors !== undefined) this.useColors = config.colors;
+		if (config.prefix) this.prefix = config.prefix;
+		if (config.logger) this.instance = config.logger;
+	}
+
+	static getInstance(): LoggerInterface {
+		return this.instance || new Logger();
+	}
+
+	static debug(message: string, ...args: any[]): void {
+		this.log("debug", message, args);
+	}
+
+	static info(message: string, ...args: any[]): void {
+		this.log("info", message, args);
+	}
+
+	static warn(message: string, ...args: any[]): void {
+		this.log("warn", message, args);
+	}
+
+	static error(message: string, ...args: any[]): void {
+		this.log("error", message, args);
+	}
+
+	private static shouldLog(level: LogLevel): boolean {
+		if (this.logLevel === "silent") return false;
+
+		const levels: LogMethod[] = ["error", "warn", "info", "debug"];
+		return levels.indexOf(level as LogMethod) <= levels.indexOf(this.logLevel as LogMethod);
+	}
+
+	private static log(level: LogMethod, message: string, args: any[]): void {
+		if (!this.shouldLog(level)) return;
+
+		const logger = this.getInstance();
+		const formatted = this.formatMessage(level, message);
+
+		logger[level](formatted, ...args);
+	}
+
+	private static formatMessage(level: LogMethod, message: string): string {
+		const timestamp = new Date().toISOString();
+		const levelColor = this.getLevelColor(level);
+		const messageColor = this.useColors ? "\x1b[37m" : "";
+
+		return [
+			this.useColors ? "\x1b[90m" : "",
+			`${this.prefix} `,
+			`${timestamp} `,
+			levelColor,
+			`[${level.toUpperCase()}]`,
+			this.useColors ? "\x1b[0m" : "",
+			messageColor,
+			` ${message}`,
+			this.useColors ? "\x1b[0m" : "",
+		].join("");
+	}
+
+	private static getLevelColor(level: LogMethod): string {
+		if (!this.useColors) return "";
+
+		return {
+			error: "\x1b[31m", // Red
+			warn: "\x1b[33m", // Yellow
+			info: "\x1b[36m", // Cyan
+			debug: "\x1b[35m", // Magenta
+		}[level];
+	}
+
+	// Instance methods to implement LoggerInterface
+	debug(message: string, ...args: any[]): void {
+		console.debug(message, ...args);
+	}
+
+	info(message: string, ...args: any[]): void {
+		console.log(message, ...args);
+	}
+
+	warn(message: string, ...args: any[]): void {
+		console.warn(message, ...args);
+	}
+
+	error(message: string, ...args: any[]): void {
+		console.error(message, ...args);
+	}
+}

package/src/utils/string-utils.ts

@@ -0,0 +1,9 @@
+export const normalizeString = (str: string): string => {
+	return str
+		.trim()
+		.toLowerCase()
+		.normalize("NFD")
+		.replace(/[\u0300-\u036f]/g, "")
+		.replace(/\s+/g, "-")
+		.replace(/[^a-z0-9.-]/g, "");
+};