text-to-map 1.1.13 → 1.1.16

package/dist/bin/cities.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/bin/cities.js

@@ -0,0 +1,9 @@
+import { importCities } from "../open-data-sync/cities";
+import { defaultBinOptions } from "./constants";
+importCities(defaultBinOptions)
+    .then(() => {
+    console.log("Completed");
+})
+    .catch((error) => {
+    console.log(error);
+});

package/dist/bin/parse-ordinance.js

@@ -36,7 +36,6 @@ function main() {
         };
         console.time("downloadAndImportAllLatestAddressPoints");
         const { municipality } = yield getNewMunicipality("Česká Lípa");
-        console.log(municipality.founder.schools[0].locations);
         const addressPoints = yield parseOrdinanceToAddressPoints(lines, {
         // currentMunicipality: municipality,
         }, reportErrors, reportWarnings);

package/dist/db/address-points.js

@@ -11,7 +11,7 @@ import { AddressPointType, createSingleLineAddress } from "czech-address";
 import { SeriesType, isNegativeSeriesSpec, isRange, isSeriesSpecArray, } from "../street-markdown/types";
 import { findClosestString } from "../utils/helpers";
 import jtsk2wgs84 from "../utils/jtsk2wgs84";
-import { extractKeyValuesPairs, generate2DPlaceholders, getKnexDb, insertMultipleRows, isMysql, isSqlite, nonEmptyOrNull, } from "./db";
+import { extractKeyValuesPairs, generate2DPlaceholders, getKnexDb, insertMultipleRows, isMysql, isSqlite, nonEmptyOrNull, rawQuery, } from "./db";
 import { getFounderCityCode } from "./founders";
 import { MunicipalityType } from "./types";
 const DescriptiveType = "č.p.";
@@ -80,7 +80,7 @@ export const commitAddressPoints = (buffer) => __awaiter(void 0, void 0, void 0,
     });
     const placeHolders = generate2DPlaceholders(columnNames.length, buffer.length);
     const knex = getKnexDb();
-    yield knex.raw(`INSERT ${isMysql(knex) ? "IGNORE" : ""} INTO address_point (${columnNames.join(",")}) VALUES ${placeHolders} ${!isMysql(knex) ? "ON CONFLICT (id) DO NOTHING" : ""}`, params);
+    yield rawQuery(`INSERT ${isMysql(knex) ? "IGNORE" : ""} INTO address_point (${columnNames.join(",")}) VALUES ${placeHolders} ${!isMysql(knex) ? "ON CONFLICT (id) DO NOTHING" : ""}`, params);
     return buffer.length;
 });
 export const insertCities = (buffer) => __awaiter(void 0, void 0, void 0, function* () {
@@ -129,8 +129,7 @@ const addressPointSelect = `
   LEFT JOIN prague_district p ON a.prague_district_code = p.code
   LEFT JOIN municipality_part m ON a.municipality_part_code = m.code`;
 export const getAddressPointById = (addressPointId) => __awaiter(void 0, void 0, void 0, function* () {
-    const knex = getKnexDb();
-    const row = yield knex.raw(`${addressPointSelect}
+    const row = yield rawQuery(`${addressPointSelect}
       WHERE a.id = ?`, [addressPointId]);
     if (row.length === 0) {
         return null;
@@ -144,7 +143,7 @@ export const checkStreetExists = (streetName, founder) => __awaiter(void 0, void
     const errors = [];
     // we check the whole city
     const cityCode = yield getFounderCityCode(founder);
-    const rowList = yield knex.raw(`SELECT name AS street_name
+    const rowList = yield rawQuery(`SELECT name AS street_name
     FROM street
     WHERE city_code = ? AND name = ?  ${isSqlite(knex) ? "COLLATE NOCASE" : ""}`, [cityCode, streetName]);
     if (rowList.length > 0) {
@@ -207,7 +206,7 @@ export const findAddressPoints = (params) => __awaiter(void 0, void 0, void 0, f
             (params.type === FindAddressPointsType.WholeMunicipalityNoStreetName
                 ? " AND a.street_code IS NULL"
                 : "");
-    const queryResult = yield knex.raw(`SELECT a.id, s.name AS street_name, o.name AS object_type_name, a.descriptive_number,
+    const queryResult = yield rawQuery(`SELECT a.id, s.name AS street_name, o.name AS object_type_name, a.descriptive_number,
       a.orientational_number, a.orientational_number_letter, c.name AS city_name,
       d.name AS district_name, m.name AS municipality_part_name, p.name AS prague_district_name,
       a.postal_code, a.wgs84_latitude, a.wgs84_longitude

package/dist/db/cities.d.ts

@@ -0,0 +1,2 @@
+export declare const insertCityPositions: (data: string[][]) => Promise<number>;
+export declare const insertCities: (buffer: string[][]) => Promise<number>;

package/dist/db/cities.js

@@ -0,0 +1,48 @@
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import chunk from "lodash/chunk";
+import { extractKeyValuesPairs, insertMultipleRows, rawQuery } from "./db";
+const citiesColumn = {
+    cityName: 0,
+    cityCode: 1,
+    countyName: 2,
+    countyCode: 3,
+    regionName: 4,
+    regionCode: 5,
+    postalCode: 6,
+    latitude: 7,
+    longitude: 8,
+};
+export const insertCityPositions = (data) => __awaiter(void 0, void 0, void 0, function* () {
+    let changes = 0;
+    // cities are most likely already inserted, but in case they're not,
+    // we need to insert them before updating them with region data
+    changes += yield insertCities(data);
+    const prev = changes;
+    for (const arrayChunk of chunk(data, 1000)) {
+        const queries = [];
+        for (const row of arrayChunk) {
+            queries.push(rawQuery(`UPDATE city SET wgs84_latitude = ?, wgs84_longitude = ? WHERE code = ?`, [
+                row[citiesColumn.latitude],
+                row[citiesColumn.longitude],
+                row[citiesColumn.cityCode],
+            ]));
+        }
+        yield Promise.all(queries);
+        changes += queries.length;
+        console.log(`Done ${changes - prev} / ${data.length} rows...`);
+    }
+    return changes;
+});
+export const insertCities = (buffer) => __awaiter(void 0, void 0, void 0, function* () {
+    return yield insertMultipleRows(extractKeyValuesPairs(buffer, citiesColumn.cityCode, [
+        citiesColumn.cityName,
+    ]), "city", ["code", "name"]);
+});

package/dist/db/db.d.ts

@@ -20,6 +20,7 @@ export declare const initDb: (config?: Partial<DbConfig>) => Promise<Knex>;
 export declare const clearDb: (config?: Partial<DbConfig>) => Promise<Knex>;
 export declare const disconnectKnex: () => Promise<void>;
 export declare const nonEmptyOrNull: (value: string) => string | null;
+export declare const rawQuery: (query: string, ...bindings: readonly Knex.RawBinding[]) => Promise<any[]>;
 /**
  * Efficiently insert multiple rows. If preventDuplicatesByFirstColumn is true, the first
  * column should be unique (PK or UNIQUE).

package/dist/db/db.js

@@ -57,6 +57,7 @@ export const getKnexDb = (config = {}) => {
                 client: DbClient[SupportedDbType.postgres],
                 connection: _knexDbConfig.pgConnectionString,
                 useNullAsDefault: true,
+                debug: _knexDbConfig.verbose,
             });
         }
         else if (_knexDbConfig.dbType === SupportedDbType.mysql) {
@@ -71,6 +72,7 @@ export const getKnexDb = (config = {}) => {
                     database,
                 },
                 useNullAsDefault: true,
+                debug: _knexDbConfig.verbose,
             });
         }
         else {
@@ -141,6 +143,11 @@ const getEnvConfig = () => {
 export const nonEmptyOrNull = (value) => {
     return value ? value : null;
 };
+export const rawQuery = (query, ...bindings) => __awaiter(void 0, void 0, void 0, function* () {
+    const knex = getKnexDb();
+    const result = yield knex.raw(query, ...bindings);
+    return isMysql(knex) ? result[0] : result;
+});
 /**
  * Efficiently insert multiple rows. If preventDuplicatesByFirstColumn is true, the first
  * column should be unique (PK or UNIQUE).
@@ -152,7 +159,7 @@ export const insertMultipleRows = (rows, table, columnNames, preventDuplicates =
     const insertPlaceholders = generate2DPlaceholders(columnNames.length, rows.length);
     const knex = getKnexDb();
     const onConfict = preventDuplicates && !isMysql(knex) ? `ON CONFLICT DO NOTHING` : "";
-    yield knex.raw(`INSERT ${isMysql(knex) ? "IGNORE" : ""} INTO ${table} (${columnNames.join(",")}) VALUES ${insertPlaceholders} ${onConfict}`, rows.flat());
+    yield rawQuery(`INSERT ${isMysql(knex) ? "IGNORE" : ""} INTO ${table} (${columnNames.join(",")}) VALUES ${insertPlaceholders} ${onConfict}`, rows.flat());
     return rows.length;
 });
 /**

package/dist/db/migrations/20230731232915_city_positions.js

@@ -0,0 +1,25 @@
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+export function up(knex) {
+    return __awaiter(this, void 0, void 0, function* () {
+        yield knex.schema.alterTable("city", (t) => {
+            t.double("wgs84_latitude").nullable();
+            t.double("wgs84_longitude").nullable();
+        });
+    });
+}
+export function down(knex) {
+    return __awaiter(this, void 0, void 0, function* () {
+        yield knex.schema.alterTable("city", (t) => {
+            t.dropColumn("wgs84_latitude");
+            t.dropColumn("wgs84_longitude");
+        });
+    });
+}

package/dist/db/types.d.ts

@@ -47,5 +47,6 @@ export declare enum SyncPart {
     AddressPoints = "address-points",
     Schools = "schools",
     Regions = "regions",
-    Streets = "streets"
+    Streets = "streets",
+    Cities = "cities"
 }

package/dist/db/types.js

@@ -16,4 +16,5 @@ export var SyncPart;
     SyncPart["Schools"] = "schools";
     SyncPart["Regions"] = "regions";
     SyncPart["Streets"] = "streets";
+    SyncPart["Cities"] = "cities";
 })(SyncPart || (SyncPart = {}));

package/dist/index.d.ts

@@ -1,11 +1,12 @@
 import { clearDb, initDb } from "./db/db";
 import { downloadAndImportAddressPoints } from "./open-data-sync/address-points";
+import { importCities } from "./open-data-sync/cities";
 import { downloadAndImportRegions } from "./open-data-sync/regions";
 import { downloadAndImportSchools } from "./open-data-sync/schools";
 import { downloadAndImportStreets } from "./open-data-sync/streets";
 import { OpenDataSyncOptionsPartial } from "./utils/helpers";
-export * from "./street-markdown/types";
 export * from "./street-markdown/smd";
+export * from "./street-markdown/types";
 /**
  * Download and import all open data to SQLite DB. The DB file will be located
  * in the `dataDir` folder.
@@ -18,4 +19,4 @@ export * from "./street-markdown/smd";
  * ```
  */
 export declare function downloadAndImportEverything(options?: OpenDataSyncOptionsPartial, syncStreets?: boolean): Promise<void>;
-export { initDb, downloadAndImportAddressPoints, clearDb, downloadAndImportSchools, downloadAndImportStreets, downloadAndImportRegions, };
+export { clearDb, downloadAndImportAddressPoints, downloadAndImportRegions, downloadAndImportSchools, downloadAndImportStreets, importCities, initDb, };

package/dist/index.js

@@ -8,12 +8,13 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
     });
 };
 import { clearDb, initDb } from "./db/db";
-import { downloadAndImportAddressPoints, } from "./open-data-sync/address-points";
+import { downloadAndImportAddressPoints } from "./open-data-sync/address-points";
+import { importCities } from "./open-data-sync/cities";
 import { downloadAndImportRegions } from "./open-data-sync/regions";
 import { downloadAndImportSchools } from "./open-data-sync/schools";
 import { downloadAndImportStreets } from "./open-data-sync/streets";
-export * from "./street-markdown/types";
 export * from "./street-markdown/smd";
+export * from "./street-markdown/types";
 /**
  * Download and import all open data to SQLite DB. The DB file will be located
  * in the `dataDir` folder.
@@ -31,9 +32,10 @@ export function downloadAndImportEverything(options = {}, syncStreets = true) {
         yield downloadAndImportAddressPoints(options);
         yield downloadAndImportSchools(options);
         yield downloadAndImportRegions(options);
+        yield importCities(options);
         if (syncStreets) {
             yield downloadAndImportStreets(options);
         }
     });
 }
-export { initDb, downloadAndImportAddressPoints, clearDb, downloadAndImportSchools, downloadAndImportStreets, downloadAndImportRegions, };
+export { clearDb, downloadAndImportAddressPoints, downloadAndImportRegions, downloadAndImportSchools, downloadAndImportStreets, importCities, initDb, };

package/dist/open-data-sync/cities.d.ts

@@ -0,0 +1,2 @@
+import { OpenDataSyncOptionsPartial } from "../utils/helpers";
+export declare const importCities: (options?: OpenDataSyncOptionsPartial) => Promise<void>;

package/dist/open-data-sync/cities.js

@@ -0,0 +1,40 @@
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import { parse } from "csv-parse";
+import iconv from "iconv-lite";
+import { pipeline } from "stream/promises";
+import { SyncPart } from "../db/types";
+import { prepareOptions, } from "../utils/helpers";
+import { runSyncPart } from "./common";
+import { createReadStream } from "fs";
+import { insertCityPositions } from "../db/cities";
+import { fileURLToPath } from "url";
+import { dirname, join } from "path";
+const importDataToDb = (options) => __awaiter(void 0, void 0, void 0, function* () {
+    console.log("Starting to parse CSV file...");
+    const __filename = fileURLToPath(import.meta.url);
+    const __dirname = dirname(__filename);
+    // source: https://github.com/33bcdd/souradnice-mest/blob/master/souradnice.csv
+    const csvStream = createReadStream(join(__dirname, "..", "souradnice.csv"));
+    const rows = [];
+    const parseStream = parse({ delimiter: ",", fromLine: 2 }).on("data", (data) => {
+        rows.push(data);
+    });
+    yield pipeline(csvStream, iconv.decodeStream("utf-8"), parseStream);
+    console.log("Parsing completed. Starting to import data to DB...");
+    yield insertCityPositions(rows);
+    console.log(`Import completed. Total imported rows: ${rows.length}`);
+});
+export const importCities = (options = {}) => __awaiter(void 0, void 0, void 0, function* () {
+    yield runSyncPart(SyncPart.Regions, [], () => __awaiter(void 0, void 0, void 0, function* () {
+        const completeOptions = prepareOptions(options);
+        yield importDataToDb(completeOptions);
+    }));
+});