postal-code-scraper 1.0.3 → 1.0.4

package/dist/scraper/scrapers.js

@@ -1,149 +0,0 @@
-"use strict";
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.PostalCodeScraper = void 0;
-const path_1 = __importDefault(require("path"));
-const puppeteer_1 = __importDefault(require("puppeteer"));
-const queue_1 = require("./queue");
-const fetchers_1 = require("./fetchers");
-const id_generator_1 = require("../utils/id-generator");
-const fs_1 = require("fs");
-const cheerio_1 = require("cheerio");
-const parsers_1 = require("./parsers");
-const env_config_1 = require("../utils/env-config");
-const string_utils_1 = require("../utils/string-utils");
-const logger_1 = require("../utils/logger");
-class PostalCodeScraper {
-    constructor(config = {}) {
-        this.config = config;
-        this.config = Object.assign({ concurrency: 15, maxRetries: 5, headless: true, directory: "src/data", logger: logger_1.Logger, usePrettyName: false }, config);
-    }
-    scrapeCountry(countryName) {
-        return __awaiter(this, void 0, void 0, function* () {
-            var _a;
-            yield this.initBrowser();
-            try {
-                const country = yield this.getCountryDetails(countryName);
-                if (!country) {
-                    (_a = this.config.logger) === null || _a === void 0 ? void 0 : _a.warn(`Country not found: ${countryName}`);
-                    return null;
-                }
-                const data = {};
-                yield this.queue.process(country, data);
-                this.saveData(data, `${country.name}-postal-codes.json`, this.config.directory);
-                const postalCodeLookup = this.generatePostalCodeLookup(data);
-                this.saveData(postalCodeLookup, `${country.name}-lookup.json`, this.config.directory);
-            }
-            finally {
-                yield this.cleanup();
-            }
-        });
-    }
-    scrapeCountries() {
-        return __awaiter(this, void 0, void 0, function* () {
-            var _a, _b;
-            yield this.initBrowser();
-            try {
-                const countries = yield this.getCountriesDetails();
-                if (countries.length === 0) {
-                    (_a = this.config.logger) === null || _a === void 0 ? void 0 : _a.warn("No countries found.");
-                    return null;
-                }
-                for (const country of countries) {
-                    const key = this.config.usePrettyName ? country.prettyName : country.name;
-                    const countryData = {};
-                    (_b = this.config.logger) === null || _b === void 0 ? void 0 : _b.info(`Processing country: ${key}`);
-                    yield this.queue.process(country, countryData);
-                    this.saveData(countryData, `${key}-postal-codes.json`, this.config.directory);
-                    const postalCodeLookup = this.generatePostalCodeLookup(countryData);
-                    this.saveData(postalCodeLookup, `${key}-lookup.json`, this.config.directory);
-                }
-            }
-            finally {
-                yield this.cleanup();
-            }
-        });
-    }
-    initBrowser() {
-        return __awaiter(this, void 0, void 0, function* () {
-            this.browser = yield puppeteer_1.default.launch({ headless: this.config.headless });
-            this.fetcher = new fetchers_1.Fetcher(this.browser, this.config);
-            this.queue = new queue_1.ProcessingQueue(this.fetcher, this.config);
-        });
-    }
-    getCountryDetails(name) {
-        return __awaiter(this, void 0, void 0, function* () {
-            var _a;
-            try {
-                const html = yield this.fetcher.fetchWithRetry((0, env_config_1.getBaseUrl)());
-                return parsers_1.Parser.parseCountryByName((0, cheerio_1.load)(html), this.config, name);
-            }
-            catch (error) {
-                (_a = this.config.logger) === null || _a === void 0 ? void 0 : _a.error(`Error fetching country details: ${name}`, error);
-                return null;
-            }
-        });
-    }
-    getCountriesDetails() {
-        return __awaiter(this, void 0, void 0, function* () {
-            var _a;
-            try {
-                const html = yield this.fetcher.fetchWithRetry((0, env_config_1.getBaseUrl)());
-                return parsers_1.Parser.parseCountries((0, cheerio_1.load)(html), this.config);
-            }
-            catch (error) {
-                (_a = this.config.logger) === null || _a === void 0 ? void 0 : _a.error("Error fetching countries details", error);
-                return [];
-            }
-        });
-    }
-    generatePostalCodeLookup(data) {
-        return this.buildLookup(data, (0, id_generator_1.createRegionIdGenerator)());
-    }
-    buildLookup(regionObj, idGenerator, acc = [], result = { postalCodeMap: {}, regions: {} }) {
-        if (Array.isArray(regionObj)) {
-            for (const item of regionObj) {
-                const id = idGenerator(acc);
-                result.postalCodeMap[item] = id;
-                result.regions[id] = [...acc];
-            }
-        }
-        else if (typeof regionObj === "object" && regionObj !== null) {
-            for (const [regionKey, regionValue] of Object.entries(regionObj)) {
-                this.buildLookup(regionValue, idGenerator, [...acc, regionKey], result);
-            }
-        }
-        return result;
-    }
-    saveData(data, fileName, directory = "src/data") {
-        var _a, _b;
-        try {
-            (0, fs_1.mkdirSync)(directory, { recursive: true });
-            const filePath = path_1.default.join(directory, (0, string_utils_1.normalizeString)(fileName));
-            (0, fs_1.writeFileSync)(filePath, JSON.stringify(data, null, 2), { flag: "w" });
-            (_a = this.config.logger) === null || _a === void 0 ? void 0 : _a.info(`Saved data to ${filePath}`);
-        }
-        catch (error) {
-            (_b = this.config.logger) === null || _b === void 0 ? void 0 : _b.error(`Error saving data to ${fileName}`, error);
-        }
-    }
-    cleanup() {
-        return __awaiter(this, void 0, void 0, function* () {
-            var _a;
-            yield ((_a = this.browser) === null || _a === void 0 ? void 0 : _a.close());
-        });
-    }
-}
-exports.PostalCodeScraper = PostalCodeScraper;
-exports.default = PostalCodeScraper;

package/dist/types.d.ts

@@ -1,32 +0,0 @@
-export type Region = {
-    path: string;
-    name: string;
-    prettyName: string;
-};
-export type ScraperConfig = {
-    usePrettyName?: boolean;
-    directory?: string;
-    concurrency?: number;
-    maxRetries?: number;
-    headless?: boolean;
-    logger?: any;
-};
-export type ProcessingQueueItem = {
-    region: Region;
-    currData: RegionData;
-};
-export interface LookupData {
-    postalCodeMap: {
-        [postalCode: string]: string;
-    };
-    regions: {
-        [code: string]: string[];
-    };
-}
-export interface PostalCodeData {
-    rawData: RegionData;
-    postalCodeLookup: LookupData;
-}
-export interface RegionData {
-    [key: string]: RegionData | string[];
-}

package/dist/types.js

@@ -1,2 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });

package/dist/utils/env-config.d.ts

@@ -1 +0,0 @@
-export declare const getBaseUrl: () => string;

package/dist/utils/env-config.js

@@ -1,7 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.getBaseUrl = void 0;
-const getBaseUrl = () => {
-    return "https://worldpostalcode.com";
-};
-exports.getBaseUrl = getBaseUrl;

package/dist/utils/id-generator.d.ts

@@ -1,4 +0,0 @@
-export interface RegionIdGenerator {
-    (regions: string[]): string;
-}
-export declare const createRegionIdGenerator: () => RegionIdGenerator;

package/dist/utils/id-generator.js

@@ -1,26 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.createRegionIdGenerator = void 0;
-const createRegionIdGenerator = () => {
-    const regionRegistry = new Map();
-    const counterMap = new Map();
-    return (regions) => {
-        const normalized = regions.map((region) => region
-            .trim()
-            .toLowerCase()
-            .normalize("NFD")
-            .replace(/[\u0300-\u036f]/g, "")
-            .replace(/\s+/g, "_"));
-        const compositeKey = normalized.join("|");
-        if (regionRegistry.has(compositeKey)) {
-            return regionRegistry.get(compositeKey);
-        }
-        const baseName = normalized[normalized.length - 1];
-        const count = (counterMap.get(baseName) || 0) + 1;
-        counterMap.set(baseName, count);
-        const newId = `${baseName}_${count}`;
-        regionRegistry.set(compositeKey, newId);
-        return newId;
-    };
-};
-exports.createRegionIdGenerator = createRegionIdGenerator;

package/dist/utils/logger.d.ts

@@ -1,33 +0,0 @@
-export type LogMethod = "error" | "warn" | "info" | "debug";
-export type LogLevel = LogMethod | "silent";
-export interface LoggerInterface {
-    debug(message: string, ...args: any[]): void;
-    info(message: string, ...args: any[]): void;
-    warn(message: string, ...args: any[]): void;
-    error(message: string, ...args: any[]): void;
-}
-export declare class Logger implements LoggerInterface {
-    private static logLevel;
-    private static useColors;
-    private static prefix;
-    private static instance;
-    static configure(config: {
-        level?: LogLevel;
-        colors?: boolean;
-        prefix?: string;
-        logger?: LoggerInterface;
-    }): void;
-    static getInstance(): LoggerInterface;
-    static debug(message: string, ...args: any[]): void;
-    static info(message: string, ...args: any[]): void;
-    static warn(message: string, ...args: any[]): void;
-    static error(message: string, ...args: any[]): void;
-    private static shouldLog;
-    private static log;
-    private static formatMessage;
-    private static getLevelColor;
-    debug(message: string, ...args: any[]): void;
-    info(message: string, ...args: any[]): void;
-    warn(message: string, ...args: any[]): void;
-    error(message: string, ...args: any[]): void;
-}

package/dist/utils/logger.js

@@ -1,86 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.Logger = void 0;
-class Logger {
-    static configure(config) {
-        if (config.level)
-            this.logLevel = config.level;
-        if (config.colors !== undefined)
-            this.useColors = config.colors;
-        if (config.prefix)
-            this.prefix = config.prefix;
-        if (config.logger)
-            this.instance = config.logger;
-    }
-    static getInstance() {
-        return this.instance || new Logger();
-    }
-    static debug(message, ...args) {
-        this.log("debug", message, args);
-    }
-    static info(message, ...args) {
-        this.log("info", message, args);
-    }
-    static warn(message, ...args) {
-        this.log("warn", message, args);
-    }
-    static error(message, ...args) {
-        this.log("error", message, args);
-    }
-    static shouldLog(level) {
-        if (this.logLevel === "silent")
-            return false;
-        const levels = ["error", "warn", "info", "debug"];
-        return levels.indexOf(level) <= levels.indexOf(this.logLevel);
-    }
-    static log(level, message, args) {
-        if (!this.shouldLog(level))
-            return;
-        const logger = this.getInstance();
-        const formatted = this.formatMessage(level, message);
-        logger[level](formatted, ...args);
-    }
-    static formatMessage(level, message) {
-        const timestamp = new Date().toISOString();
-        const levelColor = this.getLevelColor(level);
-        const messageColor = this.useColors ? "\x1b[37m" : "";
-        return [
-            this.useColors ? "\x1b[90m" : "",
-            `${this.prefix} `,
-            `${timestamp} `,
-            levelColor,
-            `[${level.toUpperCase()}]`,
-            this.useColors ? "\x1b[0m" : "",
-            messageColor,
-            ` ${message}`,
-            this.useColors ? "\x1b[0m" : "",
-        ].join("");
-    }
-    static getLevelColor(level) {
-        if (!this.useColors)
-            return "";
-        return {
-            error: "\x1b[31m", // Red
-            warn: "\x1b[33m", // Yellow
-            info: "\x1b[36m", // Cyan
-            debug: "\x1b[35m", // Magenta
-        }[level];
-    }
-    // Instance methods to implement LoggerInterface
-    debug(message, ...args) {
-        console.debug(message, ...args);
-    }
-    info(message, ...args) {
-        console.log(message, ...args);
-    }
-    warn(message, ...args) {
-        console.warn(message, ...args);
-    }
-    error(message, ...args) {
-        console.error(message, ...args);
-    }
-}
-exports.Logger = Logger;
-Logger.logLevel = "info";
-Logger.useColors = true;
-Logger.prefix = "[POSTAL-CODE-SCRAPER]";

package/dist/utils/string-utils.d.ts

@@ -1 +0,0 @@
-export declare const normalizeString: (str: string) => string;

package/dist/utils/string-utils.js

@@ -1,13 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.normalizeString = void 0;
-const normalizeString = (str) => {
-    return str
-        .trim()
-        .toLowerCase()
-        .normalize("NFD")
-        .replace(/[\u0300-\u036f]/g, "")
-        .replace(/\s+/g, "-")
-        .replace(/[^a-z0-9.-]/g, "");
-};
-exports.normalizeString = normalizeString;

package/src/index.ts

@@ -1,2 +0,0 @@
-export * from "./types";
-export { PostalCodeScraper } from "./scraper/scrapers";

package/src/scraper/fetchers.ts

@@ -1,30 +0,0 @@
-import { Browser } from "puppeteer";
-import { ScraperConfig } from "../types";
-
-export class Fetcher {
-	constructor(private browser: Browser, private config: ScraperConfig) {}
-
-	async fetchHtml(url: string): Promise<string> {
-		const page = await this.browser.newPage();
-		try {
-			page.setDefaultNavigationTimeout(60000);
-			await page.goto(url, { waitUntil: "domcontentloaded" });
-			return await page.content();
-		} finally {
-			await page.close();
-		}
-	}
-
-	async fetchWithRetry(url: string, retries = this.config.maxRetries || 5): Promise<string> {
-		try {
-			return await this.fetchHtml(url);
-		} catch (error) {
-			this.config.logger?.warn(`Retrying (${this.config.maxRetries! - retries + 1}) for: ${url}`);
-			if (retries > 0) {
-				await new Promise((resolve) => setTimeout(resolve, Math.random() * 7000 + 5000));
-				return this.fetchWithRetry(url, retries - 1);
-			}
-			throw new Error(`Failed to fetch: ${url} after ${this.config.maxRetries} attempts`);
-		}
-	}
-}

package/src/scraper/parsers.ts

@@ -1,67 +0,0 @@
-import { Region, ScraperConfig } from "../types";
-
-export class Parser {
-	static parseRegions($: cheerio.Root, config: ScraperConfig): Region[] {
-		return $("h2:contains('Regions')")
-			.next(".regions")
-			.find("a")
-			.map((_index, element) => {
-				const path = $(element).attr("href");
-				const prettyName = $(element).text().trim();
-				if (!path || !prettyName) return null;
-
-				return {
-					name: path.split("/").filter(Boolean).pop()!,
-					prettyName,
-					path,
-				};
-			})
-			.get()
-			.filter(Boolean) as Region[];
-	}
-
-	static parsePostalCodes($: cheerio.Root, config: ScraperConfig): Record<string, string[]> {
-		const codes: Record<string, string[]> = {};
-
-		$(".codes .container").each((_i, element) => {
-			const place = $(element).find(".place").text().trim();
-			const codesList = $(element)
-				.find(".code span")
-				.map((_j, el) => $(el).text().trim())
-				.get();
-
-			if (place) {
-				const key = config.usePrettyName ? place : place.toLowerCase().replace(/\s+/g, "-");
-				codes[key] = codesList;
-			}
-		});
-
-		return codes;
-	}
-
-	static parseCountries($: cheerio.Root, config: ScraperConfig): Region[] {
-		return $(".regions div a")
-			.map((_i, element) => {
-				const path = $(element).attr("href");
-				return path ? { name: path.replace(/\//g, ""), prettyName: $(element).text().trim(), path } : null;
-			})
-			.get()
-			.filter(Boolean) as Region[];
-	}
-
-	static parseCountryByName($: cheerio.Root, config: ScraperConfig, name: string): Region | null {
-		const countryElement = $(`.regions div a`).filter((_, el) => $(el).attr("href")?.replace(/\//g, "") === name.toLowerCase().trim());
-
-		if (!countryElement.length) return null;
-
-		const path = countryElement.attr("href");
-		const prettyName = countryElement.text().trim();
-		return path && prettyName
-			? {
-					name: path.replace(/\//g, ""),
-					prettyName,
-					path,
-			  }
-			: null;
-	}
-}

package/src/scraper/queue.ts

@@ -1,55 +0,0 @@
-import { load } from "cheerio";
-import { Region, ProcessingQueueItem, ScraperConfig, RegionData } from "../types";
-import { Fetcher } from "./fetchers";
-import pLimit from "p-limit";
-import { Parser } from "./parsers";
-import { getBaseUrl } from "../utils/env-config";
-
-export class ProcessingQueue {
-	private queue: ProcessingQueueItem[] = [];
-	private visitedUrls = new Set<string>();
-	private limit: ReturnType<typeof pLimit>;
-
-	constructor(private fetcher: Fetcher, private config: ScraperConfig) {
-		this.limit = pLimit(config.concurrency || 15);
-	}
-
-	async process(startRegion: Region, data: RegionData): Promise<void> {
-		this.queue.push({ region: startRegion, currData: data });
-
-		while (this.queue.length > 0) {
-			const tasks = this.queue.map((item) => this.limit(() => this.processItem(item)));
-			this.queue = [];
-			await Promise.all(tasks);
-		}
-	}
-
-	private async processItem(item: ProcessingQueueItem): Promise<void> {
-		const url = `${getBaseUrl()}${item.region.path}`;
-
-		if (this.visitedUrls.has(url)) return;
-		this.visitedUrls.add(url);
-
-		this.config.logger?.info(`Fetching: ${url}`);
-
-		try {
-			const html = await this.fetcher.fetchWithRetry(url);
-			const $ = load(html);
-
-			const regions = Parser.parseRegions($, this.config);
-			regions.forEach((region) => {
-				const key = this.config.usePrettyName ? region.prettyName : region.name;
-				item.currData[key] = {};
-				this.queue.push({
-					region,
-					currData: item.currData[key],
-				});
-			});
-
-			const codes = Parser.parsePostalCodes($, this.config);
-			Object.assign(item.currData, codes);
-		} catch (error) {
-			this.config.logger?.error(`Error processing ${url}:`, error);
-		}
-	}
-}

package/src/scraper/scrapers.ts

@@ -1,143 +0,0 @@
-import path from "path";
-import puppeteer, { Browser } from "puppeteer";
-import { ProcessingQueue } from "./queue";
-import { Fetcher } from "./fetchers";
-import { Region, ScraperConfig, LookupData, RegionData } from "../types";
-import { createRegionIdGenerator, RegionIdGenerator } from "../utils/id-generator";
-import { writeFileSync, mkdirSync } from "fs";
-import { load } from "cheerio";
-import { Parser } from "./parsers";
-import { getBaseUrl } from "../utils/env-config";
-import { normalizeString } from "../utils/string-utils";
-import { Logger } from "../utils/logger";
-
-export class PostalCodeScraper {
-	private browser!: Browser;
-	private queue!: ProcessingQueue;
-	private fetcher!: Fetcher;
-
-	constructor(private config: ScraperConfig = {}) {
-		this.config = {
-			concurrency: 15,
-			maxRetries: 5,
-			headless: true,
-			directory: "src/data",
-			logger: Logger,
-			usePrettyName: false,
-			...config,
-		};
-	}
-
-	async scrapeCountry(countryName: string) {
-		await this.initBrowser();
-		try {
-			const country = await this.getCountryDetails(countryName);
-			if (!country) {
-				this.config.logger?.warn(`Country not found: ${countryName}`);
-				return null;
-			}
-
-			const data: RegionData = {};
-			await this.queue.process(country, data);
-
-			this.saveData(data, `${country.name}-postal-codes.json`, this.config.directory!);
-
-			const postalCodeLookup = this.generatePostalCodeLookup(data);
-			this.saveData(postalCodeLookup, `${country.name}-lookup.json`, this.config.directory!);
-		} finally {
-			await this.cleanup();
-		}
-	}
-
-	async scrapeCountries() {
-		await this.initBrowser();
-		try {
-			const countries = await this.getCountriesDetails();
-			if (countries.length === 0) {
-				this.config.logger?.warn("No countries found.");
-				return null;
-			}
-
-			for (const country of countries) {
-				const key = this.config.usePrettyName ? country.prettyName : country.name;
-				const countryData: RegionData = {};
-				this.config.logger?.info(`Processing country: ${key}`);
-
-				await this.queue.process(country, countryData);
-				this.saveData(countryData, `${key}-postal-codes.json`, this.config.directory!);
-
-				const postalCodeLookup = this.generatePostalCodeLookup(countryData);
-				this.saveData(postalCodeLookup, `${key}-lookup.json`, this.config.directory!);
-			}
-		} finally {
-			await this.cleanup();
-		}
-	}
-
-	private async initBrowser() {
-		this.browser = await puppeteer.launch({ headless: this.config.headless });
-		this.fetcher = new Fetcher(this.browser, this.config);
-		this.queue = new ProcessingQueue(this.fetcher, this.config);
-	}
-
-	private async getCountryDetails(name: string): Promise<Region | null> {
-		try {
-			const html = await this.fetcher.fetchWithRetry(getBaseUrl());
-			return Parser.parseCountryByName(load(html), this.config, name);
-		} catch (error) {
-			this.config.logger?.error(`Error fetching country details: ${name}`, error);
-			return null;
-		}
-	}
-
-	private async getCountriesDetails(): Promise<Region[]> {
-		try {
-			const html = await this.fetcher.fetchWithRetry(getBaseUrl());
-			return Parser.parseCountries(load(html), this.config);
-		} catch (error) {
-			this.config.logger?.error("Error fetching countries details", error);
-			return [];
-		}
-	}
-
-	private generatePostalCodeLookup(data: RegionData): LookupData {
-		return this.buildLookup(data, createRegionIdGenerator());
-	}
-
-	private buildLookup(
-		regionObj: RegionData | string[],
-		idGenerator: RegionIdGenerator,
-		acc: string[] = [],
-		result: LookupData = { postalCodeMap: {}, regions: {} }
-	): LookupData {
-		if (Array.isArray(regionObj)) {
-			for (const item of regionObj) {
-				const id = idGenerator(acc);
-				result.postalCodeMap[item] = id;
-				result.regions[id] = [...acc];
-			}
-		} else if (typeof regionObj === "object" && regionObj !== null) {
-			for (const [regionKey, regionValue] of Object.entries(regionObj)) {
-				this.buildLookup(regionValue, idGenerator, [...acc, regionKey], result);
-			}
-		}
-		return result;
-	}
-
-	private saveData(data: any, fileName: string, directory: string = "src/data") {
-		try {
-			mkdirSync(directory, { recursive: true });
-			const filePath = path.join(directory, normalizeString(fileName));
-			writeFileSync(filePath, JSON.stringify(data, null, 2), { flag: "w" });
-			this.config.logger?.info(`Saved data to ${filePath}`);
-		} catch (error) {
-			this.config.logger?.error(`Error saving data to ${fileName}`, error);
-		}
-	}
-
-	private async cleanup() {
-		await this.browser?.close();
-	}
-}
-
-export default PostalCodeScraper;