text-to-map 1.0.5 → 1.0.6

package/dist/bin/address-points.js

@@ -1,6 +1,6 @@
 import { downloadAndImportAddressPoints } from "../open-data-sync/address-points";
 console.time("downloadAndImportAllLatestAddressPoints");
-downloadAndImportAddressPoints()
+downloadAndImportAddressPoints({ dataDir: "./tmp/db" })
     .then(() => {
     console.timeEnd("downloadAndImportAllLatestAddressPoints");
 })

package/dist/bin/regions.js

@@ -1,5 +1,5 @@
 import { downloadAndImportRegions } from "../open-data-sync/regions";
-downloadAndImportRegions()
+downloadAndImportRegions({ dataDir: "./tmp/db" })
     .then(() => {
     console.log("Completed");
 })

package/dist/bin/schools.js

@@ -1,5 +1,5 @@
 import { downloadAndImportSchools } from "../open-data-sync/schools";
-downloadAndImportSchools({ tmpDir: "tmp" })
+downloadAndImportSchools({ dataDir: "./tmp/db" })
     .then(() => {
     console.log("Completed");
 })

package/dist/bin/streets.js

@@ -1,6 +1,6 @@
 import { downloadAndImportStreets } from "../open-data-sync/streets";
 console.time("downloadAndImportAllStreets");
-downloadAndImportStreets({})
+downloadAndImportStreets({ dataDir: "./tmp/db" })
     .then(() => {
     console.timeEnd("downloadAndImportAllStreets");
 })

package/dist/bin/sync-all.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/bin/sync-all.js

@@ -0,0 +1,2 @@
+import { downloadAndImportEverything } from "../index";
+downloadAndImportEverything({ dataDir: "./tmp/db" });

package/dist/db/regions.d.ts

@@ -1,4 +1,4 @@
-export declare const Column: {
+interface RegionsColumn {
     cityName: number;
     cityCode: number;
     cityType: number;
@@ -15,9 +15,17 @@ export declare const Column: {
     regionCsuCode100: number;
     regionCsuCode108Nuts: number;
     regionRuianCode: number;
-};
-export declare const insertRegionsAndOrps: (data: string[][]) => number;
-export declare const insertRegions: (buffer: string[][]) => number;
-export declare const insertCounties: (buffer: string[][]) => number;
-export declare const insertOrps: (buffer: string[][]) => number;
-export declare const insertCities: (buffer: string[][]) => number;
+}
+export interface RegionsTableSchema {
+    tableSchema: {
+        columns: {
+            name: string;
+        }[];
+    };
+}
+export declare const insertRegionsAndOrps: (data: string[][], schema: RegionsTableSchema) => number;
+export declare const insertRegions: (buffer: string[][], columnIndex: RegionsColumn) => number;
+export declare const insertCounties: (buffer: string[][], columnIndex: RegionsColumn) => number;
+export declare const insertOrps: (buffer: string[][], columnIndex: RegionsColumn) => number;
+export declare const insertCities: (buffer: string[][], columnIndex: RegionsColumn) => number;
+export {};

package/dist/db/regions.js

@@ -1,30 +1,13 @@
 import { extractKeyValuesPairs, getDb, insertMultipleRows } from "./db";
-export const Column = {
-    cityName: 0,
-    cityCode: 1,
-    cityType: 2,
-    orpName: 9,
-    orpCsuCode65: 10,
-    orpRuianCode: 11,
-    orpCityCode: 12,
-    countyName: 13,
-    countyCsuCode101Lau: 14,
-    countyCsuCode109Nuts: 15,
-    countyRuianCode: 16,
-    regionName: 17,
-    regionShortName: 18,
-    regionCsuCode100: 19,
-    regionCsuCode108Nuts: 20,
-    regionRuianCode: 21,
-};
-export const insertRegionsAndOrps = (data) => {
+export const insertRegionsAndOrps = (data, schema) => {
+    const columnIndex = getColumnIndexFromSchema(schema);
     let changes = 0;
-    changes += insertRegions(data);
-    changes += insertCounties(data);
-    changes += insertOrps(data);
+    changes += insertRegions(data, columnIndex);
+    changes += insertCounties(data, columnIndex);
+    changes += insertOrps(data, columnIndex);
     // cities are most likely already inserted, but in case they're not,
     // we need to insert them before updating them with region data
-    changes += insertCities(data);
+    changes += insertCities(data, columnIndex);
     const db = getDb();
     const updateStatement = db.prepare(`UPDATE city SET
       region_code = ?,
@@ -32,34 +15,63 @@ export const insertRegionsAndOrps = (data) => {
       orp_code = ?
     WHERE code = ?`);
     data.forEach((data) => {
-        changes += updateStatement.run(data[Column.regionRuianCode], data[Column.countyRuianCode], data[Column.orpRuianCode], data[Column.cityCode]).changes;
+        changes += updateStatement.run(data[columnIndex.regionRuianCode], data[columnIndex.countyRuianCode], data[columnIndex.orpRuianCode], data[columnIndex.cityCode]).changes;
     });
     return changes;
 };
-export const insertRegions = (buffer) => {
-    return insertMultipleRows(extractKeyValuesPairs(buffer, Column.regionRuianCode, [
-        Column.regionName,
-        Column.regionShortName,
-        Column.regionCsuCode100,
-        Column.regionCsuCode108Nuts,
+export const insertRegions = (buffer, columnIndex) => {
+    return insertMultipleRows(extractKeyValuesPairs(buffer, columnIndex.regionRuianCode, [
+        columnIndex.regionName,
+        columnIndex.regionShortName,
+        columnIndex.regionCsuCode100,
+        columnIndex.regionCsuCode108Nuts,
     ]), "region", ["code", "name", "short_name", "csu_code_100", "csu_code_108_nuts"]);
 };
-export const insertCounties = (buffer) => {
-    return insertMultipleRows(extractKeyValuesPairs(buffer, Column.countyRuianCode, [
-        Column.countyName,
-        Column.countyCsuCode101Lau,
-        Column.countyCsuCode109Nuts,
-        Column.regionRuianCode,
+export const insertCounties = (buffer, columnIndex) => {
+    return insertMultipleRows(extractKeyValuesPairs(buffer, columnIndex.countyRuianCode, [
+        columnIndex.countyName,
+        columnIndex.countyCsuCode101Lau,
+        columnIndex.countyCsuCode109Nuts,
+        columnIndex.regionRuianCode,
     ]), "county", ["code", "name", "csu_code_101_lau", "csu_code_109_nuts", "region_code"]);
 };
-export const insertOrps = (buffer) => {
-    return insertMultipleRows(extractKeyValuesPairs(buffer, Column.orpRuianCode, [
-        Column.orpName,
-        Column.orpCsuCode65,
-        Column.regionRuianCode,
-        Column.countyRuianCode,
+export const insertOrps = (buffer, columnIndex) => {
+    return insertMultipleRows(extractKeyValuesPairs(buffer, columnIndex.orpRuianCode, [
+        columnIndex.orpName,
+        columnIndex.orpCsuCode65,
+        columnIndex.regionRuianCode,
+        columnIndex.countyRuianCode,
     ]), "orp", ["code", "name", "csu_code_65", "region_code", "county_code"]);
 };
-export const insertCities = (buffer) => {
-    return insertMultipleRows(extractKeyValuesPairs(buffer, Column.cityCode, [Column.cityName]), "city", ["code", "name"]);
+export const insertCities = (buffer, columnIndex) => {
+    return insertMultipleRows(extractKeyValuesPairs(buffer, columnIndex.cityCode, [columnIndex.cityName]), "city", ["code", "name"]);
+};
+const getColumnIndexFromSchema = (schema) => {
+    const names = schema.tableSchema.columns.map((column) => column.name);
+    const columnIndex = {
+        cityName: findOrDie(names, "obec_text"),
+        cityCode: findOrDie(names, "obec_kod"),
+        cityType: findOrDie(names, "obec_typ"),
+        orpName: findOrDie(names, "orp_text"),
+        orpCsuCode65: findOrDie(names, "orp_csu_cis65_kod"),
+        orpRuianCode: findOrDie(names, "orp_ruian_kod"),
+        orpCityCode: findOrDie(names, "orp_sidlo_obec_kod"),
+        countyName: findOrDie(names, "okres_text"),
+        countyCsuCode101Lau: findOrDie(names, "okres_csu_cis101_lau_kod"),
+        countyCsuCode109Nuts: findOrDie(names, "okres_csu_cis109_nuts_kod"),
+        countyRuianCode: findOrDie(names, "okres_ruian_kod"),
+        regionName: findOrDie(names, "kraj_text"),
+        regionShortName: findOrDie(names, "kraj_zkratka"),
+        regionCsuCode100: findOrDie(names, "kraj_csu_cis100_kod"),
+        regionCsuCode108Nuts: findOrDie(names, "kraj_csu_cis108_nuts_kod"),
+        regionRuianCode: findOrDie(names, "kraj_ruian_vusc_kod"),
+    };
+    return columnIndex;
+};
+const findOrDie = (columnNames, name) => {
+    const index = columnNames.indexOf(name);
+    if (index < 0) {
+        throw new Error(`Column '${name}' not found in region schema. Check https://data.gov.cz/datov%C3%A1-sada?iri=https%3A%2F%2Fdata.gov.cz%2Fzdroj%2Fdatov%C3%A9-sady%2F00025593%2F271b0fb7c2abb7f44e12ad57617821b2 for current dataset info or search for 'Struktura území ČR se všemi kódy území od obcí po stát dle číselníků ČSÚ, klasifikace NUTS a kódů RÚIAN.'.`);
+    }
+    return index;
 };

package/dist/open-data-sync/regions.js

@@ -25,9 +25,15 @@ const downloadAndImportDataToDb = (options) => __awaiter(void 0, void 0, void 0,
         rows.push(data);
     });
     yield pipeline(response.body, iconv.decodeStream("utf-8"), parseStream);
+    const schemaResponse = yield fetch(options.regionsSchemaUrl);
+    if (schemaResponse.status !== 200) {
+        throw new Error(`The file could not be downloaded. HTTP Code: ${schemaResponse.status}`);
+    }
+    // @ts-ignore
+    const schema = yield schemaResponse.json();
     console.log("Parsing completed. Starting to import data to DB...");
     initDb(options);
-    insertRegionsAndOrps(rows);
+    insertRegionsAndOrps(rows, schema);
     console.log(`Import completed. Total imported rows: ${rows.length}`);
 });
 export const downloadAndImportRegions = (options = {}) => __awaiter(void 0, void 0, void 0, function* () {

package/dist/utils/helpers.d.ts

@@ -13,6 +13,7 @@ export interface OpenDataSyncOptions {
     schoolsXmlUrl?: string;
     schoolsXmlFileName?: string;
     regionsCsvUrl?: string;
+    regionsSchemaUrl?: string;
     regionsCsvFileName?: string;
 }
 export interface OpenDataSyncOptionsNotEmpty {
@@ -29,6 +30,7 @@ export interface OpenDataSyncOptionsNotEmpty {
     schoolsXmlUrl: string;
     schoolsXmlFileName: string;
     regionsCsvUrl: string;
+    regionsSchemaUrl: string;
     regionsCsvFileName: string;
 }
 export declare const getAppDataDirPath: () => string;

package/dist/utils/helpers.js

@@ -10,7 +10,7 @@ export const getAppDataDirPath = () => join(process.env.APPDATA ||
         ? process.env.HOME + "/Library/Preferences"
         : process.env.HOME + "/.local/share"), appName);
 export const prepareOptions = (options) => {
-    var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o, _p;
+    var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o, _p, _q;
     const dataDir = (_a = options.dataDir) !== null && _a !== void 0 ? _a : getAppDataDirPath();
     if (!existsSync(dataDir)) {
         mkdirSync(dataDir);
@@ -35,7 +35,8 @@ export const prepareOptions = (options) => {
         schoolsXmlUrl: (_l = options.schoolsXmlUrl) !== null && _l !== void 0 ? _l : "https://rejstriky.msmt.cz/opendata/vrejcelk.xml",
         schoolsXmlFileName: (_m = options.schoolsXmlFileName) !== null && _m !== void 0 ? _m : "school-register.xml",
         regionsCsvUrl: (_o = options.regionsCsvUrl) !== null && _o !== void 0 ? _o : "https://www.czso.cz/documents/10180/23208674/struktura_uzemi_cr.csv",
-        regionsCsvFileName: (_p = options.regionsCsvFileName) !== null && _p !== void 0 ? _p : "struktura_uzemi_cr.csv",
+        regionsSchemaUrl: (_p = options.regionsSchemaUrl) !== null && _p !== void 0 ? _p : "https://www.czso.cz/documents/10180/23208674/struktura_uzemi_cr-metadata.json",
+        regionsCsvFileName: (_q = options.regionsCsvFileName) !== null && _q !== void 0 ? _q : "struktura_uzemi_cr.csv",
     };
 };
 export const initDb = (options) => {

package/package.json

@@ -29,14 +29,14 @@
     "ts-node": "^10.9.1"
   },
   "scripts": {
+    "sync-all": "ts-node --experimental-specifier-resolution=node --esm src/bin/sync-all.ts",
     "address-points": "ts-node --experimental-specifier-resolution=node --esm src/bin/address-points.ts",
+    "regions": "ts-node --experimental-specifier-resolution=node --esm src/bin/regions.ts",
     "schools": "ts-node --experimental-specifier-resolution=node --esm src/bin/schools.ts",
+    "streets": "ts-node --experimental-specifier-resolution=node --esm src/bin/streets.ts",
+    "diagrams": "ts-node --experimental-specifier-resolution=node --esm src/bin/generate-syntax-diagrams.ts",
     "clear-all": "ts-node --experimental-specifier-resolution=node --esm src/bin/clear-all.ts",
     "parse-ordinance": "ts-node --experimental-specifier-resolution=node --esm src/bin/parse-ordinance.ts",
-    "diagrams": "ts-node --experimental-specifier-resolution=node --esm src/bin/generate-syntax-diagrams.ts",
-    "streets": "ts-node --experimental-specifier-resolution=node --esm src/bin/streets.ts",
-    "regions": "ts-node --experimental-specifier-resolution=node --esm src/bin/regions.ts",
-    "dbf": "ts-node --experimental-specifier-resolution=node --esm src/bin/dbf.ts",
     "test": "node --experimental-vm-modules --no-warnings node_modules/jest/bin/jest.js -c=jest-esm.config.mjs --no-cache",
     "ordinance-to-csv-rules": "node src/smd-to-csv/ordinance-to-csv-rules.js",
     "clean": "rimraf dist",
@@ -48,7 +48,7 @@
   },
   "name": "text-to-map",
   "description": "Text To Map usiluje o lepší, strojově zpracovatelné využití částí vyhlášek s výčtem ulic a dalších lokací. Jde o rozšiřitelnou sadu konceptů a nástrojů, které zajistí hladký převod výčtu ulic a jejich rozsahů v lidsky srozumitelném jazyce do strojově zpracovatelného, uchopitelného formátu.",
-  "version": "1.0.5",
+  "version": "1.0.6",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "files": [