text-to-map 1.1.12 → 1.1.15

package/dist/bin/cities.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/bin/cities.js

@@ -0,0 +1,9 @@
+import { importCities } from "../open-data-sync/cities";
+import { defaultBinOptions } from "./constants";
+importCities(defaultBinOptions)
+    .then(() => {
+    console.log("Completed");
+})
+    .catch((error) => {
+    console.log(error);
+});

package/dist/db/cities.d.ts

@@ -0,0 +1,2 @@
+export declare const insertCityPositions: (data: string[][]) => Promise<number>;
+export declare const insertCities: (buffer: string[][]) => Promise<number>;

package/dist/db/cities.js

@@ -0,0 +1,49 @@
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import chunk from "lodash/chunk";
+import { extractKeyValuesPairs, getKnexDb, insertMultipleRows } from "./db";
+const citiesColumn = {
+    cityName: 0,
+    cityCode: 1,
+    countyName: 2,
+    countyCode: 3,
+    regionName: 4,
+    regionCode: 5,
+    postalCode: 6,
+    latitude: 7,
+    longitude: 8,
+};
+export const insertCityPositions = (data) => __awaiter(void 0, void 0, void 0, function* () {
+    let changes = 0;
+    // cities are most likely already inserted, but in case they're not,
+    // we need to insert them before updating them with region data
+    changes += yield insertCities(data);
+    const prev = changes;
+    const knex = getKnexDb();
+    for (const arrayChunk of chunk(data, 1000)) {
+        const queries = [];
+        for (const row of arrayChunk) {
+            queries.push(knex.raw(`UPDATE city SET wgs84_latitude = ?, wgs84_longitude = ? WHERE code = ?`, [
+                row[citiesColumn.latitude],
+                row[citiesColumn.longitude],
+                row[citiesColumn.cityCode],
+            ]));
+        }
+        yield Promise.all(queries);
+        changes += queries.length;
+        console.log(`Done ${changes - prev} / ${data.length} rows...`);
+    }
+    return changes;
+});
+export const insertCities = (buffer) => __awaiter(void 0, void 0, void 0, function* () {
+    return yield insertMultipleRows(extractKeyValuesPairs(buffer, citiesColumn.cityCode, [
+        citiesColumn.cityName,
+    ]), "city", ["code", "name"]);
+});

package/dist/db/migrations/20230731232915_city_positions.js

@@ -0,0 +1,25 @@
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+export function up(knex) {
+    return __awaiter(this, void 0, void 0, function* () {
+        yield knex.schema.alterTable("city", (t) => {
+            t.double("wgs84_latitude").nullable();
+            t.double("wgs84_longitude").nullable();
+        });
+    });
+}
+export function down(knex) {
+    return __awaiter(this, void 0, void 0, function* () {
+        yield knex.schema.alterTable("city", (t) => {
+            t.dropColumn("wgs84_latitude");
+            t.dropColumn("wgs84_longitude");
+        });
+    });
+}

package/dist/db/types.d.ts

@@ -47,5 +47,6 @@ export declare enum SyncPart {
     AddressPoints = "address-points",
     Schools = "schools",
     Regions = "regions",
-    Streets = "streets"
+    Streets = "streets",
+    Cities = "cities"
 }

package/dist/db/types.js

@@ -16,4 +16,5 @@ export var SyncPart;
     SyncPart["Schools"] = "schools";
     SyncPart["Regions"] = "regions";
     SyncPart["Streets"] = "streets";
+    SyncPart["Cities"] = "cities";
 })(SyncPart || (SyncPart = {}));

package/dist/index.d.ts

@@ -1,11 +1,12 @@
 import { clearDb, initDb } from "./db/db";
 import { downloadAndImportAddressPoints } from "./open-data-sync/address-points";
+import { importCities } from "./open-data-sync/cities";
 import { downloadAndImportRegions } from "./open-data-sync/regions";
 import { downloadAndImportSchools } from "./open-data-sync/schools";
 import { downloadAndImportStreets } from "./open-data-sync/streets";
 import { OpenDataSyncOptionsPartial } from "./utils/helpers";
-export * from "./street-markdown/types";
 export * from "./street-markdown/smd";
+export * from "./street-markdown/types";
 /**
  * Download and import all open data to SQLite DB. The DB file will be located
  * in the `dataDir` folder.
@@ -18,4 +19,4 @@ export * from "./street-markdown/smd";
  * ```
  */
 export declare function downloadAndImportEverything(options?: OpenDataSyncOptionsPartial, syncStreets?: boolean): Promise<void>;
-export { initDb, downloadAndImportAddressPoints, clearDb, downloadAndImportSchools, downloadAndImportStreets, downloadAndImportRegions, };
+export { clearDb, downloadAndImportAddressPoints, downloadAndImportRegions, downloadAndImportSchools, downloadAndImportStreets, importCities, initDb, };

package/dist/index.js

@@ -8,12 +8,13 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
     });
 };
 import { clearDb, initDb } from "./db/db";
-import { downloadAndImportAddressPoints, } from "./open-data-sync/address-points";
+import { downloadAndImportAddressPoints } from "./open-data-sync/address-points";
+import { importCities } from "./open-data-sync/cities";
 import { downloadAndImportRegions } from "./open-data-sync/regions";
 import { downloadAndImportSchools } from "./open-data-sync/schools";
 import { downloadAndImportStreets } from "./open-data-sync/streets";
-export * from "./street-markdown/types";
 export * from "./street-markdown/smd";
+export * from "./street-markdown/types";
 /**
  * Download and import all open data to SQLite DB. The DB file will be located
  * in the `dataDir` folder.
@@ -31,9 +32,10 @@ export function downloadAndImportEverything(options = {}, syncStreets = true) {
         yield downloadAndImportAddressPoints(options);
         yield downloadAndImportSchools(options);
         yield downloadAndImportRegions(options);
+        yield importCities(options);
         if (syncStreets) {
             yield downloadAndImportStreets(options);
         }
     });
 }
-export { initDb, downloadAndImportAddressPoints, clearDb, downloadAndImportSchools, downloadAndImportStreets, downloadAndImportRegions, };
+export { clearDb, downloadAndImportAddressPoints, downloadAndImportRegions, downloadAndImportSchools, downloadAndImportStreets, importCities, initDb, };

package/dist/open-data-sync/address-points.js

@@ -14,6 +14,7 @@ import iconv from "iconv-lite";
 import fetch from "node-fetch";
 import { join } from "path";
 import { pipeline } from "stream/promises";
+import chunk from "lodash/chunk";
 import { commitAddressPoints } from "../db/address-points";
 import { SyncPart } from "../db/types";
 import { getLatestUrlFromAtomFeed } from "../utils/atom";
@@ -42,24 +43,21 @@ const importDataToDb = (options) => __awaiter(void 0, void 0, void 0, function*
     let total = 0;
     let next = 0;
     console.log("Initiating import of RUIAN data to search DB (~3 million rows to be imported).");
-    const buffer = [];
     for (const file of files) {
+        const rows = [];
         const parseStream = parse({ delimiter: ";", fromLine: 2 }).on("data", (data) => __awaiter(void 0, void 0, void 0, function* () {
-            buffer.push(data);
-            if (buffer.length >= maxBufferSize) {
-                parseStream.pause();
-                total += yield commitAddressPoints(buffer);
-                if (total - next >= 100000) {
-                    next += 100000;
-                    console.log(`Total imported rows: ${next}`);
-                }
-                buffer.length = 0;
-                parseStream.resume();
-            }
+            rows.push(data);
         }));
         yield pipeline(createReadStream(join(extractionFolder, file)), iconv.decodeStream("win1250"), parseStream);
+        const chunks = chunk(rows, maxBufferSize);
+        yield Promise.all(chunks.map((chunk) => __awaiter(void 0, void 0, void 0, function* () {
+            total += yield commitAddressPoints(chunk);
+            if (total - next >= 100000) {
+                next += 100000;
+                console.log(`Total imported rows: ${next}`);
+            }
+        })));
     }
-    total += yield commitAddressPoints(buffer);
     console.log(`Import completed. Total imported rows: ${total}`);
 });
 const getExtractionFolder = (options) => join(options.tmpDir, options.addressPointsCsvFolderName);

package/dist/open-data-sync/cities.d.ts

@@ -0,0 +1,2 @@
+import { OpenDataSyncOptionsPartial } from "../utils/helpers";
+export declare const importCities: (options?: OpenDataSyncOptionsPartial) => Promise<void>;

package/dist/open-data-sync/cities.js

@@ -0,0 +1,40 @@
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import { parse } from "csv-parse";
+import iconv from "iconv-lite";
+import { pipeline } from "stream/promises";
+import { SyncPart } from "../db/types";
+import { prepareOptions, } from "../utils/helpers";
+import { runSyncPart } from "./common";
+import { createReadStream } from "fs";
+import { insertCityPositions } from "../db/cities";
+import { fileURLToPath } from "url";
+import { dirname, join } from "path";
+const importDataToDb = (options) => __awaiter(void 0, void 0, void 0, function* () {
+    console.log("Starting to parse CSV file...");
+    const __filename = fileURLToPath(import.meta.url);
+    const __dirname = dirname(__filename);
+    // source: https://github.com/33bcdd/souradnice-mest/blob/master/souradnice.csv
+    const csvStream = createReadStream(join(__dirname, "..", "souradnice.csv"));
+    const rows = [];
+    const parseStream = parse({ delimiter: ",", fromLine: 2 }).on("data", (data) => {
+        rows.push(data);
+    });
+    yield pipeline(csvStream, iconv.decodeStream("utf-8"), parseStream);
+    console.log("Parsing completed. Starting to import data to DB...");
+    yield insertCityPositions(rows);
+    console.log(`Import completed. Total imported rows: ${rows.length}`);
+});
+export const importCities = (options = {}) => __awaiter(void 0, void 0, void 0, function* () {
+    yield runSyncPart(SyncPart.Regions, [], () => __awaiter(void 0, void 0, void 0, function* () {
+        const completeOptions = prepareOptions(options);
+        yield importDataToDb(completeOptions);
+    }));
+});