postal-code-scraper 1.0.3 → 1.0.4

package/dist/index.js

@@ -1,20 +1,355 @@
-"use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __exportStar = (this && this.__exportStar) || function(m, exports) {
-    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.PostalCodeScraper = void 0;
-__exportStar(require("./types"), exports);
-var scrapers_1 = require("./scraper/scrapers");
-Object.defineProperty(exports, "PostalCodeScraper", { enumerable: true, get: function () { return scrapers_1.PostalCodeScraper; } });
+// src/scraper/scrapers.ts
+import path from "path";
+import puppeteer from "puppeteer";
+
+// src/scraper/queue.ts
+import { load } from "cheerio";
+import pLimit from "p-limit";
+
+// src/scraper/parsers.ts
+var Parser = class {
+  static parseRegions($, config) {
+    return $("h2:contains('Regions')").next(".regions").find("a").map((_index, element) => {
+      const path2 = $(element).attr("href");
+      const prettyName = $(element).text().trim();
+      if (!path2 || !prettyName) return null;
+      return {
+        name: path2.split("/").filter(Boolean).pop(),
+        prettyName,
+        path: path2
+      };
+    }).get().filter(Boolean);
+  }
+  static parsePostalCodes($, config) {
+    const codes = {};
+    $(".codes .container").each((_i, element) => {
+      const place = $(element).find(".place").text().trim();
+      const codesList = $(element).find(".code span").map((_j, el) => $(el).text().trim()).get();
+      if (place) {
+        const key = config.usePrettyName ? place : place.toLowerCase().replace(/\s+/g, "-");
+        codes[key] = codesList;
+      }
+    });
+    return codes;
+  }
+  static parseCountries($, config) {
+    return $(".regions div a").map((_i, element) => {
+      const path2 = $(element).attr("href");
+      return path2 ? { name: path2.replace(/\//g, ""), prettyName: $(element).text().trim(), path: path2 } : null;
+    }).get().filter(Boolean);
+  }
+  static parseCountryByName($, config, name) {
+    const countryElement = $(`.regions div a`).filter((_, el) => $(el).attr("href")?.replace(/\//g, "") === name.toLowerCase().trim());
+    if (!countryElement.length) return null;
+    const path2 = countryElement.attr("href");
+    const prettyName = countryElement.text().trim();
+    return path2 && prettyName ? {
+      name: path2.replace(/\//g, ""),
+      prettyName,
+      path: path2
+    } : null;
+  }
+};
+
+// src/utils/env-config.ts
+var getBaseUrl = () => {
+  return "https://worldpostalcode.com";
+};
+
+// src/scraper/queue.ts
+var ProcessingQueue = class {
+  constructor(fetcher, config) {
+    this.fetcher = fetcher;
+    this.config = config;
+    this.limit = pLimit(config.concurrency || 15);
+  }
+  queue = [];
+  visitedUrls = /* @__PURE__ */ new Set();
+  limit;
+  async process(startRegion, data) {
+    this.queue.push({ region: startRegion, currData: data });
+    while (this.queue.length > 0) {
+      const tasks = this.queue.map((item) => this.limit(() => this.processItem(item)));
+      this.queue = [];
+      await Promise.all(tasks);
+    }
+  }
+  async processItem(item) {
+    const url = `${getBaseUrl()}${item.region.path}`;
+    if (this.visitedUrls.has(url)) return;
+    this.visitedUrls.add(url);
+    this.config.logger?.info(`Fetching: ${url}`);
+    try {
+      const html = await this.fetcher.fetchWithRetry(url);
+      const $ = load(html);
+      const regions = Parser.parseRegions($, this.config);
+      regions.forEach((region) => {
+        const key = this.config.usePrettyName ? region.prettyName : region.name;
+        item.currData[key] = {};
+        this.queue.push({
+          region,
+          currData: item.currData[key]
+        });
+      });
+      const codes = Parser.parsePostalCodes($, this.config);
+      Object.assign(item.currData, codes);
+    } catch (error) {
+      this.config.logger?.error(`Error processing ${url}:`, error);
+    }
+  }
+};
+
+// src/scraper/fetchers.ts
+var Fetcher = class {
+  constructor(browser, config) {
+    this.browser = browser;
+    this.config = config;
+  }
+  async fetchHtml(url) {
+    const page = await this.browser.newPage();
+    try {
+      page.setDefaultNavigationTimeout(6e4);
+      await page.goto(url, { waitUntil: "domcontentloaded" });
+      return await page.content();
+    } finally {
+      await page.close();
+    }
+  }
+  async fetchWithRetry(url, retries = this.config.maxRetries || 5) {
+    try {
+      return await this.fetchHtml(url);
+    } catch (error) {
+      this.config.logger?.warn(`Retrying (${this.config.maxRetries - retries + 1}) for: ${url}`);
+      if (retries > 0) {
+        await new Promise((resolve) => setTimeout(resolve, Math.random() * 7e3 + 5e3));
+        return this.fetchWithRetry(url, retries - 1);
+      }
+      throw new Error(`Failed to fetch: ${url} after ${this.config.maxRetries} attempts`);
+    }
+  }
+};
+
+// src/utils/id-generator.ts
+var createRegionIdGenerator = () => {
+  const regionRegistry = /* @__PURE__ */ new Map();
+  const counterMap = /* @__PURE__ */ new Map();
+  return (regions) => {
+    const normalized = regions.map(
+      (region) => region.trim().toLowerCase().normalize("NFD").replace(/[\u0300-\u036f]/g, "").replace(/\s+/g, "_")
+    );
+    const compositeKey = normalized.join("|");
+    if (regionRegistry.has(compositeKey)) {
+      return regionRegistry.get(compositeKey);
+    }
+    const baseName = normalized[normalized.length - 1];
+    const count = (counterMap.get(baseName) || 0) + 1;
+    counterMap.set(baseName, count);
+    const newId = `${baseName}_${count}`;
+    regionRegistry.set(compositeKey, newId);
+    return newId;
+  };
+};
+
+// src/scraper/scrapers.ts
+import { writeFileSync, mkdirSync } from "fs";
+import { load as load2 } from "cheerio";
+
+// src/utils/string-utils.ts
+var normalizeString = (str) => {
+  return str.trim().toLowerCase().normalize("NFD").replace(/[\u0300-\u036f]/g, "").replace(/\s+/g, "-").replace(/[^a-z0-9.-]/g, "");
+};
+
+// src/utils/logger.ts
+var Logger = class _Logger {
+  static logLevel = "info";
+  static useColors = true;
+  static prefix = "[POSTAL-CODE-SCRAPER]";
+  static instance;
+  static configure(config) {
+    if (config.level) this.logLevel = config.level;
+    if (config.colors !== void 0) this.useColors = config.colors;
+    if (config.prefix) this.prefix = config.prefix;
+    if (config.logger) this.instance = config.logger;
+  }
+  static getInstance() {
+    return this.instance || new _Logger();
+  }
+  static debug(message, ...args) {
+    this.log("debug", message, args);
+  }
+  static info(message, ...args) {
+    this.log("info", message, args);
+  }
+  static warn(message, ...args) {
+    this.log("warn", message, args);
+  }
+  static error(message, ...args) {
+    this.log("error", message, args);
+  }
+  static shouldLog(level) {
+    if (this.logLevel === "silent") return false;
+    const levels = ["error", "warn", "info", "debug"];
+    return levels.indexOf(level) <= levels.indexOf(this.logLevel);
+  }
+  static log(level, message, args) {
+    if (!this.shouldLog(level)) return;
+    const logger = this.getInstance();
+    const formatted = this.formatMessage(level, message);
+    logger[level](formatted, ...args);
+  }
+  static formatMessage(level, message) {
+    const timestamp = (/* @__PURE__ */ new Date()).toISOString();
+    const levelColor = this.getLevelColor(level);
+    const messageColor = this.useColors ? "\x1B[37m" : "";
+    return [
+      this.useColors ? "\x1B[90m" : "",
+      `${this.prefix} `,
+      `${timestamp} `,
+      levelColor,
+      `[${level.toUpperCase()}]`,
+      this.useColors ? "\x1B[0m" : "",
+      messageColor,
+      ` ${message}`,
+      this.useColors ? "\x1B[0m" : ""
+    ].join("");
+  }
+  static getLevelColor(level) {
+    if (!this.useColors) return "";
+    return {
+      error: "\x1B[31m",
+      // Red
+      warn: "\x1B[33m",
+      // Yellow
+      info: "\x1B[36m",
+      // Cyan
+      debug: "\x1B[35m"
+      // Magenta
+    }[level];
+  }
+  // Instance methods to implement LoggerInterface
+  debug(message, ...args) {
+    console.debug(message, ...args);
+  }
+  info(message, ...args) {
+    console.log(message, ...args);
+  }
+  warn(message, ...args) {
+    console.warn(message, ...args);
+  }
+  error(message, ...args) {
+    console.error(message, ...args);
+  }
+};
+
+// src/scraper/scrapers.ts
+var PostalCodeScraper = class {
+  constructor(config = {}) {
+    this.config = config;
+    this.config = {
+      concurrency: 15,
+      maxRetries: 5,
+      headless: true,
+      directory: "src/data",
+      logger: Logger,
+      usePrettyName: false,
+      ...config
+    };
+  }
+  browser;
+  queue;
+  fetcher;
+  async scrapeCountry(countryName) {
+    await this.initBrowser();
+    try {
+      const country = await this.getCountryDetails(countryName);
+      if (!country) {
+        this.config.logger?.warn(`Country not found: ${countryName}`);
+        return null;
+      }
+      const data = {};
+      await this.queue.process(country, data);
+      this.saveData(data, `${country.name}-postal-codes.json`, this.config.directory);
+      const postalCodeLookup = this.generatePostalCodeLookup(data);
+      this.saveData(postalCodeLookup, `${country.name}-lookup.json`, this.config.directory);
+    } finally {
+      await this.cleanup();
+    }
+  }
+  async scrapeCountries() {
+    await this.initBrowser();
+    try {
+      const countries = await this.getCountriesDetails();
+      if (countries.length === 0) {
+        this.config.logger?.warn("No countries found.");
+        return null;
+      }
+      for (const country of countries) {
+        const key = this.config.usePrettyName ? country.prettyName : country.name;
+        const countryData = {};
+        this.config.logger?.info(`Processing country: ${key}`);
+        await this.queue.process(country, countryData);
+        this.saveData(countryData, `${key}-postal-codes.json`, this.config.directory);
+        const postalCodeLookup = this.generatePostalCodeLookup(countryData);
+        this.saveData(postalCodeLookup, `${key}-lookup.json`, this.config.directory);
+      }
+    } finally {
+      await this.cleanup();
+    }
+  }
+  async initBrowser() {
+    this.browser = await puppeteer.launch({ headless: this.config.headless });
+    this.fetcher = new Fetcher(this.browser, this.config);
+    this.queue = new ProcessingQueue(this.fetcher, this.config);
+  }
+  async getCountryDetails(name) {
+    try {
+      const html = await this.fetcher.fetchWithRetry(getBaseUrl());
+      return Parser.parseCountryByName(load2(html), this.config, name);
+    } catch (error) {
+      this.config.logger?.error(`Error fetching country details: ${name}`, error);
+      return null;
+    }
+  }
+  async getCountriesDetails() {
+    try {
+      const html = await this.fetcher.fetchWithRetry(getBaseUrl());
+      return Parser.parseCountries(load2(html), this.config);
+    } catch (error) {
+      this.config.logger?.error("Error fetching countries details", error);
+      return [];
+    }
+  }
+  generatePostalCodeLookup(data) {
+    return this.buildLookup(data, createRegionIdGenerator());
+  }
+  buildLookup(regionObj, idGenerator, acc = [], result = { postalCodeMap: {}, regions: {} }) {
+    if (Array.isArray(regionObj)) {
+      for (const item of regionObj) {
+        const id = idGenerator(acc);
+        result.postalCodeMap[item] = id;
+        result.regions[id] = [...acc];
+      }
+    } else if (typeof regionObj === "object" && regionObj !== null) {
+      for (const [regionKey, regionValue] of Object.entries(regionObj)) {
+        this.buildLookup(regionValue, idGenerator, [...acc, regionKey], result);
+      }
+    }
+    return result;
+  }
+  saveData(data, fileName, directory = "src/data") {
+    try {
+      mkdirSync(directory, { recursive: true });
+      const filePath = path.join(directory, normalizeString(fileName));
+      writeFileSync(filePath, JSON.stringify(data, null, 2), { flag: "w" });
+      this.config.logger?.info(`Saved data to ${filePath}`);
+    } catch (error) {
+      this.config.logger?.error(`Error saving data to ${fileName}`, error);
+    }
+  }
+  async cleanup() {
+    await this.browser?.close();
+  }
+};
+export {
+  PostalCodeScraper
+};

package/package.json

@@ -1,13 +1,16 @@
 {
 	"name": "postal-code-scraper",
-	"version": "1.0.3",
+	"version": "1.0.4",
 	"description": "A tool for scraping country data, including regions and their postal codes",
-	"main": "dist/index.js",
-	"types": "dist/index.d.ts",
+	"main": "./dist/index.cjs",
+	"module": "./dist/index.js",
+	"types": "./dist/index.d.ts",
+	"files": [
+		"dist"
+	],
 	"scripts": {
-		"start": "tsx src/index.ts",
 		"prebuild": "shx rm -rf dist",
-		"build": "tsc",
+		"build": "tsup src/index.ts --format esm,cjs --dts --clean",
 		"pretest": "npm run build",
 		"test": "mocha -r tsx tests/**/*.test.ts",
 		"prepare": "npm run build",
@@ -27,8 +30,9 @@
 		"mocha": "^11.1.0",
 		"sinon": "^19.0.2",
 		"tslib": "^2.8.1",
+		"tsup": "^8.5.1",
 		"tsx": "^3.14.0",
-		"typescript": "^5.8.2"
+		"typescript": "^5.9.3"
 	},
 	"dependencies": {
 		"cheerio": "^1.0.0",
@@ -57,5 +61,12 @@
 	},
 	"engines": {
 		"node": ">=18"
+	},
+	"exports": {
+		".": {
+			"types": "./dist/index.d.ts",
+			"import": "./dist/index.js",
+			"require": "./dist/index.cjs"
+		}
 	}
 }

package/.mocharc.json

@@ -1,4 +0,0 @@
-{
-	"extension": ["ts"],
-	"spec": "tests/**/*.test.ts"
-}

package/build/test/src/index.js

@@ -1,26 +0,0 @@
-"use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __exportStar = (this && this.__exportStar) || function(m, exports) {
-    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
-};
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.default = exports.PostalCodeScraper = void 0;
-__exportStar(require("./types"), exports);
-var scrapers_1 = require("./scraper/scrapers");
-Object.defineProperty(exports, "PostalCodeScraper", { enumerable: true, get: function () { return scrapers_1.PostalCodeScraper; } });
-var scrapers_2 = require("./scraper/scrapers");
-Object.defineProperty(exports, "default", { enumerable: true, get: function () { return __importDefault(scrapers_2).default; } });
-//# sourceMappingURL=index.js.map

package/build/test/src/index.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;AAAA,0CAAwB;AACxB,+CAAuD;AAA9C,6GAAA,iBAAiB,OAAA;AAC1B,+CAA6C;AAApC,oHAAA,OAAO,OAAA"}

package/build/test/src/scraper/fetchers.js

@@ -1,49 +0,0 @@
-"use strict";
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.Fetcher = void 0;
-class Fetcher {
-    constructor(browser, config) {
-        this.browser = browser;
-        this.config = config;
-    }
-    fetchHtml(url) {
-        return __awaiter(this, void 0, void 0, function* () {
-            const page = yield this.browser.newPage();
-            try {
-                page.setDefaultNavigationTimeout(60000);
-                yield page.goto(url, { waitUntil: "domcontentloaded" });
-                return yield page.content();
-            }
-            finally {
-                yield page.close();
-            }
-        });
-    }
-    fetchWithRetry(url_1) {
-        return __awaiter(this, arguments, void 0, function* (url, retries = this.config.maxRetries || 5) {
-            var _a;
-            try {
-                return yield this.fetchHtml(url);
-            }
-            catch (error) {
-                (_a = this.config.logger) === null || _a === void 0 ? void 0 : _a.warn(`Retrying (${this.config.maxRetries - retries + 1}) for: ${url}`);
-                if (retries > 0) {
-                    yield new Promise((resolve) => setTimeout(resolve, Math.random() * 7000 + 5000));
-                    return this.fetchWithRetry(url, retries - 1);
-                }
-                throw new Error(`Failed to fetch: ${url} after ${this.config.maxRetries} attempts`);
-            }
-        });
-    }
-}
-exports.Fetcher = Fetcher;
-//# sourceMappingURL=fetchers.js.map

package/build/test/src/scraper/fetchers.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"fetchers.js","sourceRoot":"","sources":["../../../../src/scraper/fetchers.ts"],"names":[],"mappings":";;;;;;;;;;;;AAGA,MAAa,OAAO;IACnB,YAAoB,OAAgB,EAAU,MAAqB;QAA/C,YAAO,GAAP,OAAO,CAAS;QAAU,WAAM,GAAN,MAAM,CAAe;IAAG,CAAC;IAEjE,SAAS,CAAC,GAAW;;YAC1B,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;YAC1C,IAAI,CAAC;gBACJ,IAAI,CAAC,2BAA2B,CAAC,KAAK,CAAC,CAAC;gBACxC,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,kBAAkB,EAAE,CAAC,CAAC;gBACxD,OAAO,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC;YAC7B,CAAC;oBAAS,CAAC;gBACV,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;YACpB,CAAC;QACF,CAAC;KAAA;IAEK,cAAc;6DAAC,GAAW,EAAE,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,UAAU,IAAI,CAAC;;YACtE,IAAI,CAAC;gBACJ,OAAO,MAAM,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;YAClC,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBAChB,MAAA,IAAI,CAAC,MAAM,CAAC,MAAM,0CAAE,IAAI,CAAC,aAAa,IAAI,CAAC,MAAM,CAAC,UAAW,GAAG,OAAO,GAAG,CAAC,UAAU,GAAG,EAAE,CAAC,CAAC;gBAC5F,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;oBACjB,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,MAAM,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC;oBACjF,OAAO,IAAI,CAAC,cAAc,CAAC,GAAG,EAAE,OAAO,GAAG,CAAC,CAAC,CAAC;gBAC9C,CAAC;gBACD,MAAM,IAAI,KAAK,CAAC,oBAAoB,GAAG,UAAU,IAAI,CAAC,MAAM,CAAC,UAAU,WAAW,CAAC,CAAC;YACrF,CAAC;QACF,CAAC;KAAA;CACD;AA1BD,0BA0BC"}

package/build/test/src/scraper/parsers.js

@@ -1,63 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.Parser = void 0;
-class Parser {
-    static parseRegions($, config) {
-        return $("h2:contains('Regions')")
-            .next(".regions")
-            .find("a")
-            .map((_index, element) => {
-            const path = $(element).attr("href");
-            const prettyName = $(element).text().trim();
-            if (!path || !prettyName)
-                return null;
-            return {
-                name: path.split("/").filter(Boolean).pop(),
-                prettyName,
-                path,
-            };
-        })
-            .get()
-            .filter(Boolean);
-    }
-    static parsePostalCodes($, config) {
-        const codes = {};
-        $(".codes .container").each((_i, element) => {
-            const place = $(element).find(".place").text().trim();
-            const codesList = $(element)
-                .find(".code span")
-                .map((_j, el) => $(el).text().trim())
-                .get();
-            if (place) {
-                const key = config.usePrettyName ? place : place.toLowerCase().replace(/\s+/g, "-");
-                codes[key] = codesList;
-            }
-        });
-        return codes;
-    }
-    static parseCountries($, config) {
-        return $(".regions div a")
-            .map((_i, element) => {
-            const path = $(element).attr("href");
-            return path ? { name: path.replace(/\//g, ""), prettyName: $(element).text().trim(), path } : null;
-        })
-            .get()
-            .filter(Boolean);
-    }
-    static parseCountryByName($, config, name) {
-        const countryElement = $(`.regions div a`).filter((_, el) => { var _a; return ((_a = $(el).attr("href")) === null || _a === void 0 ? void 0 : _a.replace(/\//g, "")) === name.toLowerCase().trim(); });
-        if (!countryElement.length)
-            return null;
-        const path = countryElement.attr("href");
-        const prettyName = countryElement.text().trim();
-        return path && prettyName
-            ? {
-                name: path.replace(/\//g, ""),
-                prettyName,
-                path,
-            }
-            : null;
-    }
-}
-exports.Parser = Parser;
-//# sourceMappingURL=parsers.js.map

package/build/test/src/scraper/parsers.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"parsers.js","sourceRoot":"","sources":["../../../../src/scraper/parsers.ts"],"names":[],"mappings":";;;AAEA,MAAa,MAAM;IAClB,MAAM,CAAC,YAAY,CAAC,CAAe,EAAE,MAAqB;QACzD,OAAO,CAAC,CAAC,wBAAwB,CAAC;aAChC,IAAI,CAAC,UAAU,CAAC;aAChB,IAAI,CAAC,GAAG,CAAC;aACT,GAAG,CAAC,CAAC,MAAM,EAAE,OAAO,EAAE,EAAE;YACxB,MAAM,IAAI,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACrC,MAAM,UAAU,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;YAC5C,IAAI,CAAC,IAAI,IAAI,CAAC,UAAU;gBAAE,OAAO,IAAI,CAAC;YAEtC,OAAO;gBACN,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,GAAG,EAAG;gBAC5C,UAAU;gBACV,IAAI;aACJ,CAAC;QACH,CAAC,CAAC;aACD,GAAG,EAAE;aACL,MAAM,CAAC,OAAO,CAAa,CAAC;IAC/B,CAAC;IAED,MAAM,CAAC,gBAAgB,CAAC,CAAe,EAAE,MAAqB;QAC7D,MAAM,KAAK,GAA6B,EAAE,CAAC;QAE3C,CAAC,CAAC,mBAAmB,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,OAAO,EAAE,EAAE;YAC3C,MAAM,KAAK,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;YACtD,MAAM,SAAS,GAAG,CAAC,CAAC,OAAO,CAAC;iBAC1B,IAAI,CAAC,YAAY,CAAC;iBAClB,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;iBACpC,GAAG,EAAE,CAAC;YAER,IAAI,KAAK,EAAE,CAAC;gBACX,MAAM,GAAG,GAAG,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;gBACpF,KAAK,CAAC,GAAG,CAAC,GAAG,SAAS,CAAC;YACxB,CAAC;QACF,CAAC,CAAC,CAAC;QAEH,OAAO,KAAK,CAAC;IACd,CAAC;IAED,MAAM,CAAC,cAAc,CAAC,CAAe,EAAE,MAAqB;QAC3D,OAAO,CAAC,CAAC,gBAAgB,CAAC;aACxB,GAAG,CAAC,CAAC,EAAE,EAAE,OAAO,EAAE,EAAE;YACpB,MAAM,IAAI,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACrC,OAAO,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,EAAE,UAAU,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;QACpG,CAAC,CAAC;aACD,GAAG,EAAE;aACL,MAAM,CAAC,OAAO,CAAa,CAAC;IAC/B,CAAC;IAED,MAAM,CAAC,kBAAkB,CAAC,CAAe,EAAE,MAAqB,EAAE,IAAY;QAC7E,MAAM,cAAc,GAAG,CAAC,CAAC,gBAAgB,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE,WAAC,OAAA,CAAA,MAAA,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,0CAAE,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,MAAK,IAAI,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,CAAA,EAAA,CAAC,CAAC;QAEnI,IAAI,CAAC,cAAc,CAAC,MAAM;YAAE,OAAO,IAAI,CAAC;QAExC,MAAM,IAAI,GAAG,cAAc,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACzC,MAAM,UAAU,GAAG,cAAc,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;QAChD,OAAO,IAAI,IAAI,UAAU;YACxB,CAAC,CAAC;gBACA,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC;gBAC7B,UAAU;gBACV,IAAI;aACH;YACH,CAAC,CAAC,IAAI,CAAC;IACT,CAAC;CACD;AAhED,wBAgEC"}

package/build/test/src/scraper/queue.js

@@ -1,69 +0,0 @@
-"use strict";
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.ProcessingQueue = void 0;
-const cheerio_1 = require("cheerio");
-const p_limit_1 = __importDefault(require("p-limit"));
-const parsers_1 = require("./parsers");
-class ProcessingQueue {
-    constructor(fetcher, config) {
-        this.fetcher = fetcher;
-        this.config = config;
-        this.queue = [];
-        this.visitedUrls = new Set();
-        this.limit = (0, p_limit_1.default)(config.concurrency || 15);
-    }
-    process(startRegion, data) {
-        return __awaiter(this, void 0, void 0, function* () {
-            this.queue.push({ region: startRegion, currData: data });
-            while (this.queue.length > 0) {
-                const tasks = this.queue.map((item) => this.limit(() => this.processItem(item)));
-                this.queue = [];
-                yield Promise.all(tasks);
-            }
-        });
-    }
-    processItem(item) {
-        return __awaiter(this, void 0, void 0, function* () {
-            var _a, _b;
-            const url = `${this.config.baseUrl}${item.region.path}`;
-            if (this.visitedUrls.has(url))
-                return;
-            this.visitedUrls.add(url);
-            (_a = this.config.logger) === null || _a === void 0 ? void 0 : _a.info(`Fetching: ${url}`);
-            try {
-                const html = yield this.fetcher.fetchWithRetry(url);
-                const $ = (0, cheerio_1.load)(html);
-                // Parse and add new regions to queue
-                const regions = parsers_1.Parser.parseRegions($, this.config);
-                regions.forEach((region) => {
-                    const key = this.config.usePrettyName ? region.prettyName : region.name;
-                    item.currData[key] = {};
-                    this.queue.push({
-                        region,
-                        currData: item.currData[key],
-                    });
-                });
-                // Parse postal codes
-                const codes = parsers_1.Parser.parsePostalCodes($, this.config);
-                Object.assign(item.currData, codes);
-            }
-            catch (error) {
-                (_b = this.config.logger) === null || _b === void 0 ? void 0 : _b.error(`Error processing ${url}:`, error);
-            }
-        });
-    }
-}
-exports.ProcessingQueue = ProcessingQueue;
-//# sourceMappingURL=queue.js.map

package/build/test/src/scraper/queue.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"queue.js","sourceRoot":"","sources":["../../../../src/scraper/queue.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;AAAA,qCAA+B;AAG/B,sDAA6B;AAC7B,uCAAmC;AAEnC,MAAa,eAAe;IAK3B,YAAoB,OAAgB,EAAU,MAAqB;QAA/C,YAAO,GAAP,OAAO,CAAS;QAAU,WAAM,GAAN,MAAM,CAAe;QAJ3D,UAAK,GAA0B,EAAE,CAAC;QAClC,gBAAW,GAAG,IAAI,GAAG,EAAU,CAAC;QAIvC,IAAI,CAAC,KAAK,GAAG,IAAA,iBAAM,EAAC,MAAM,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC;IAC/C,CAAC;IAEK,OAAO,CAAC,WAAmB,EAAE,IAAS;;YAC3C,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;YAEzD,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC9B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBACjF,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC;gBAChB,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YAC1B,CAAC;QACF,CAAC;KAAA;IAEa,WAAW,CAAC,IAAyB;;;YAClD,MAAM,GAAG,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;YAExD,IAAI,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC;gBAAE,OAAO;YACtC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAE1B,MAAA,IAAI,CAAC,MAAM,CAAC,MAAM,0CAAE,IAAI,CAAC,aAAa,GAAG,EAAE,CAAC,CAAC;YAE7C,IAAI,CAAC;gBACJ,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;gBACpD,MAAM,CAAC,GAAG,IAAA,cAAI,EAAC,IAAI,CAAC,CAAC;gBAErB,qCAAqC;gBACrC,MAAM,OAAO,GAAG,gBAAM,CAAC,YAAY,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;gBACpD,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE;oBAC1B,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC;oBACxE,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC;oBACxB,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;wBACf,MAAM;wBACN,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC;qBAC5B,CAAC,CAAC;gBACJ,CAAC,CAAC,CAAC;gBAEH,qBAAqB;gBACrB,MAAM,KAAK,GAAG,gBAAM,CAAC,gBAAgB,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;gBACtD,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;YACrC,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBAChB,MAAA,IAAI,CAAC,MAAM,CAAC,MAAM,0CAAE,KAAK,CAAC,oBAAoB,GAAG,GAAG,EAAE,KAAK,CAAC,CAAC;YAC9D,CAAC;QACF,CAAC;KAAA;CACD;AAjDD,0CAiDC"}