text-to-map 1.1.19 → 1.1.21

package/dist/bin/parse-ordinance.js

@@ -8,7 +8,9 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
     });
 };
 import { readFileSync, writeFileSync } from "fs";
-import { getNewMunicipalityByName, parseOrdinanceToAddressPoints, } from "../street-markdown/smd";
+import { parseOrdinanceToAddressPoints, } from "../street-markdown/smd";
+import { municipalityToPolygons } from "../street-markdown/polygons";
+import { disconnectKnex } from "../db/db";
 function main() {
     return __awaiter(this, void 0, void 0, function* () {
         // take first node argument as a file name
@@ -35,7 +37,7 @@ function main() {
             warningCount++;
         };
         console.time("downloadAndImportAllLatestAddressPoints");
-        const { municipality } = yield getNewMunicipalityByName("Česká Lípa");
+        // const { municipality } = await getNewMunicipalityByName("Česká Lípa");
         const addressPoints = yield parseOrdinanceToAddressPoints({
             lines,
             initialState: {
@@ -46,6 +48,11 @@ function main() {
             includeUnmappedAddressPoints: true,
         });
         console.timeEnd("downloadAndImportAllLatestAddressPoints");
+        const polygons = [];
+        for (const municipality of addressPoints) {
+            polygons.push(yield municipalityToPolygons(municipality));
+        }
+        yield disconnectKnex();
         console.log(`Parsed ${lines.length} lines, ${errorCount} errors, ${warningCount} warnings.`);
         if (errorCount > 0) {
             console.log("Errors:");
@@ -60,6 +67,12 @@ function main() {
         else {
             console.log(JSON.stringify(addressPoints));
         }
+        if (process.argv.length >= 5) {
+            const outputFileName = process.argv[4];
+            const output = JSON.stringify(polygons);
+            console.log(`Writing polygons to ${outputFileName}`);
+            writeFileSync(outputFileName, output);
+        }
     });
 }
 main();

package/dist/bin/polygons.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/bin/polygons.js

@@ -0,0 +1,36 @@
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import { readFileSync, writeFileSync } from "fs";
+import { municipalityToPolygons } from "../street-markdown/polygons";
+import { disconnectKnex } from "../db/db";
+function main() {
+    return __awaiter(this, void 0, void 0, function* () {
+        // take first node argument as a file name
+        if (process.argv.length < 3) {
+            console.error("Missing file name argument");
+            process.exit(1);
+        }
+        const fileName = process.argv[2];
+        const fileContent = readFileSync(fileName);
+        const json = JSON.parse(fileContent.toString());
+        const municipalityPolygons = yield municipalityToPolygons(json[0]);
+        yield disconnectKnex();
+        const output = JSON.stringify(municipalityPolygons);
+        if (process.argv.length >= 4) {
+            const outputFileName = process.argv[3];
+            console.log(`Writing polygons to ${outputFileName}`);
+            writeFileSync(outputFileName, output);
+        }
+        else {
+            console.log(JSON.stringify(output));
+        }
+    });
+}
+main();

package/dist/db/cities.d.ts

@@ -1,2 +1,5 @@
+import { FeatureCollection, MultiPolygon, Polygon } from "@turf/helpers";
 export declare const insertCityPositions: (data: string[][]) => Promise<number>;
 export declare const insertCities: (buffer: string[][]) => Promise<number>;
+export declare const setCityPolygonGeojson: (polygon: FeatureCollection, code: string) => Promise<void>;
+export declare const getCityPolygonGeojsons: (cityCodes: number[]) => Promise<Record<number, FeatureCollection<Polygon | MultiPolygon>>>;

package/dist/db/cities.js

@@ -8,7 +8,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
     });
 };
 import chunk from "lodash/chunk";
-import { extractKeyValuesPairs, insertMultipleRows, rawQuery } from "./db";
+import { extractKeyValuesPairs, getKnexDb, insertMultipleRows, rawQuery, } from "./db";
 const citiesColumn = {
     cityName: 0,
     cityCode: 1,
@@ -46,3 +46,19 @@ export const insertCities = (buffer) => __awaiter(void 0, void 0, void 0, functi
         citiesColumn.cityName,
     ]), "city", ["code", "name"]);
 });
+export const setCityPolygonGeojson = (polygon, code) => __awaiter(void 0, void 0, void 0, function* () {
+    yield getKnexDb()
+        .from("city")
+        .update({ polygon_geojson: JSON.stringify(polygon) })
+        .where({ code });
+});
+export const getCityPolygonGeojsons = (cityCodes) => __awaiter(void 0, void 0, void 0, function* () {
+    const rows = yield getKnexDb()
+        .from("city")
+        .select("code", "polygon_geojson")
+        .whereIn("code", cityCodes);
+    return rows.reduce((acc, row) => {
+        acc[row.code] = JSON.parse(row.polygon_geojson);
+        return acc;
+    }, {});
+});

package/dist/db/founders.js

@@ -284,11 +284,10 @@ export const getFounderCityCode = (founder) => __awaiter(void 0, void 0, void 0,
     if (founder.municipalityType === MunicipalityType.District) {
         if (cityCodeFounder !== founder) {
             cityCodeFounder = founder;
-            cachedCityCode = yield getKnexDb()
-                .pluck("city_code")
+            cachedCityCode = (yield getKnexDb()
                 .from("city_district")
                 .where("code", founder.cityOrDistrictCode)
-                .first();
+                .first()).city_code;
         }
         return cachedCityCode;
     }

package/dist/db/migrations/20240106224603_city_polygon.js

@@ -0,0 +1,23 @@
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+export function up(knex) {
+    return __awaiter(this, void 0, void 0, function* () {
+        yield knex.schema.alterTable("city", (t) => {
+            t.text("polygon_geojson", "longtext").nullable();
+        });
+    });
+}
+export function down(knex) {
+    return __awaiter(this, void 0, void 0, function* () {
+        yield knex.schema.alterTable("city", (t) => {
+            t.dropColumn("polygon_geojson");
+        });
+    });
+}

package/dist/index.d.ts

@@ -3,7 +3,7 @@ import { downloadAndImportAddressPoints } from "./open-data-sync/address-points"
 import { importCities } from "./open-data-sync/cities";
 import { downloadAndImportRegions } from "./open-data-sync/regions";
 import { downloadAndImportSchools } from "./open-data-sync/schools";
-import { downloadAndImportStreets } from "./open-data-sync/streets";
+import { municipalityToPolygons } from "./street-markdown/polygons";
 import { OpenDataSyncOptionsPartial } from "./utils/helpers";
 export * from "./street-markdown/smd";
 export * from "./street-markdown/types";
@@ -19,4 +19,4 @@ export * from "./street-markdown/types";
  * ```
  */
 export declare function downloadAndImportEverything(options?: OpenDataSyncOptionsPartial, syncStreets?: boolean): Promise<void>;
-export { clearDb, downloadAndImportAddressPoints, downloadAndImportRegions, downloadAndImportSchools, downloadAndImportStreets, importCities, initDb, };
+export { clearDb, downloadAndImportAddressPoints, downloadAndImportRegions, downloadAndImportSchools, importCities, municipalityToPolygons, initDb, };

package/dist/index.js

@@ -12,7 +12,8 @@ import { downloadAndImportAddressPoints } from "./open-data-sync/address-points"
 import { importCities } from "./open-data-sync/cities";
 import { downloadAndImportRegions } from "./open-data-sync/regions";
 import { downloadAndImportSchools } from "./open-data-sync/schools";
-import { downloadAndImportStreets } from "./open-data-sync/streets";
+// import { downloadAndImportStreets } from "./open-data-sync/streets";
+import { municipalityToPolygons } from "./street-markdown/polygons";
 export * from "./street-markdown/smd";
 export * from "./street-markdown/types";
 /**
@@ -34,8 +35,10 @@ export function downloadAndImportEverything(options = {}, syncStreets = true) {
         yield downloadAndImportRegions(options);
         yield importCities(options);
         if (syncStreets) {
-            yield downloadAndImportStreets(options);
+            // await downloadAndImportStreets(options);
         }
     });
 }
-export { clearDb, downloadAndImportAddressPoints, downloadAndImportRegions, downloadAndImportSchools, downloadAndImportStreets, importCities, initDb, };
+export { clearDb, downloadAndImportAddressPoints, downloadAndImportRegions, downloadAndImportSchools, 
+// downloadAndImportStreets,
+importCities, municipalityToPolygons, initDb, };

package/dist/open-data-sync/cities.js

@@ -10,13 +10,13 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
 import { parse } from "csv-parse";
 import iconv from "iconv-lite";
 import { pipeline } from "stream/promises";
+import { createReadStream } from "fs";
+import { dirname, join } from "path";
+import { fileURLToPath } from "url";
+import { insertCityPositions } from "../db/cities";
 import { SyncPart } from "../db/types";
 import { prepareOptions, } from "../utils/helpers";
 import { runSyncPart } from "./common";
-import { createReadStream } from "fs";
-import { insertCityPositions } from "../db/cities";
-import { fileURLToPath } from "url";
-import { dirname, join } from "path";
 const importDataToDb = (options) => __awaiter(void 0, void 0, void 0, function* () {
     console.log("Starting to parse CSV file...");
     const __filename = fileURLToPath(import.meta.url);

package/dist/open-data-sync/streets.js

@@ -11,6 +11,7 @@ import AdmZip from "adm-zip";
 import { createWriteStream, existsSync, mkdirSync, rmSync } from "fs";
 import fetch from "node-fetch";
 import parseDBF from "parsedbf";
+import ShpToGeoJson from "shp-to-geojson";
 import { join } from "path";
 import { pipeline } from "stream/promises";
 import { deleteStreets, getAllSyncedStreets, insertStreetsFromDbf, setStreetAsSynced, } from "../db/street-sync";
@@ -18,6 +19,9 @@ import { SyncPart } from "../db/types";
 import { getAllUrlsFromAtomFeed, getLatestUrlFromAtomFeed, } from "../utils/atom";
 import { prepareOptions, } from "../utils/helpers";
 import { runSyncPart } from "./common";
+import { coordEach } from "@turf/meta";
+import jtsk2wgs84 from "../utils/jtsk2wgs84";
+import { setCityPolygonGeojson } from "../db/cities";
 const prepareFolders = (options) => {
     const tempFolder = getTempFolder(options);
     if (!existsSync(tempFolder)) {
@@ -27,7 +31,7 @@ const prepareFolders = (options) => {
 const getTempFolder = (options) => {
     return join(options.tmpDir, options.streetZipFolderName);
 };
-const downloadZipAndParseDbfFile = (url, index, options) => __awaiter(void 0, void 0, void 0, function* () {
+const downloadZipAndParseFiles = (url, index, options) => __awaiter(void 0, void 0, void 0, function* () {
     const response = yield fetch(url);
     if (response.status !== 200) {
         throw new Error(`The ZIP file from '${url}' could not be downloaded. HTTP Code: ${response.status}`);
@@ -36,16 +40,35 @@ const downloadZipAndParseDbfFile = (url, index, options) => __awaiter(void 0, vo
     yield pipeline(response.body, createWriteStream(zipFilePath));
     const zip = new AdmZip(zipFilePath);
     const folderName = zip.getEntries()[0].entryName;
+    const cityCode = folderName.substring(0, 6);
     const dbfEntryName = `${folderName}${options.streetDbfFileName}`;
-    const obj = parseDBF(zip.getEntry(dbfEntryName).getData(), "win1250");
+    const streetData = parseDBF(zip.getEntry(dbfEntryName).getData(), "win1250");
+    const shpEntryName = `${folderName}${options.polygonShpFileName}`;
+    const polygonData = convertShpToGeoJson(zip.getEntry(shpEntryName).getData());
     rmSync(zipFilePath);
-    return obj;
+    return { streetData, polygonData, cityCode };
 });
-const importDataToDb = (data) => __awaiter(void 0, void 0, void 0, function* () {
-    if (data.length === 0) {
-        return;
+const convertShpToGeoJson = (shpBuffer) => {
+    const shp = new ShpToGeoJson({
+        arraybuffers: {
+            shpBuffer,
+        },
+    });
+    const geoJson = shp.getGeoJson();
+    delete geoJson.bbox;
+    coordEach(geoJson, (currentCoord) => {
+        const [y, x] = currentCoord;
+        const { lat, lon } = jtsk2wgs84(-x, -y);
+        currentCoord[0] = lon;
+        currentCoord[1] = lat;
+    });
+    return geoJson;
+};
+const importDataToDb = ({ streetData, polygonData, cityCode, }) => __awaiter(void 0, void 0, void 0, function* () {
+    if (streetData.length > 0) {
+        yield insertStreetsFromDbf(streetData);
     }
-    yield insertStreetsFromDbf(data);
+    yield setCityPolygonGeojson(polygonData, cityCode);
 });
 const attempts = 5;
 export const downloadAndImportStreets = (options = {}) => __awaiter(void 0, void 0, void 0, function* () {
@@ -75,8 +98,8 @@ export const downloadAndImportStreets = (options = {}) => __awaiter(void 0, void
             for (let i = 0; i < attempts; i++) {
                 try {
                     const zipLink = yield getLatestUrlFromAtomFeed(link);
-                    const dbfObject = yield downloadZipAndParseDbfFile(zipLink, done, completeOptions);
-                    yield importDataToDb(dbfObject);
+                    const data = yield downloadZipAndParseFiles(zipLink, done, completeOptions);
+                    yield importDataToDb(data);
                     yield setStreetAsSynced(link);
                     done++;
                     console.log(`Loaded links: ${done}/${allDatasetFeedLinks.length}`);