text-to-map 1.1.3 → 1.1.5

package/dist/db/address-points.js

@@ -11,7 +11,7 @@ import { AddressPointType, createSingleLineAddress } from "czech-address";
 import { SeriesType, isNegativeSeriesSpec, isRange, isSeriesSpecArray, isWholeMunicipalitySmdLine, } from "../street-markdown/types";
 import { findClosestString } from "../utils/helpers";
 import jtsk2wgs84 from "../utils/jtsk2wgs84";
-import { extractKeyValuesPairs, generate2DPlaceholders, getKnexDb, insertMultipleRows, isSqlite, nonEmptyOrNull } from "./db";
+import { extractKeyValuesPairs, generate2DPlaceholders, getKnexDb, insertMultipleRows, isSqlite, nonEmptyOrNull, } from "./db";
 import { getFounderCityCode } from "./founders";
 import { MunicipalityType } from "./types";
 const DescriptiveType = "č.p.";
@@ -108,7 +108,10 @@ export const insertStreets = (buffer) => __awaiter(void 0, void 0, void 0, funct
     ]), "street", ["code", "city_code", "name"]);
 });
 export const areAddressPointsSynced = () => __awaiter(void 0, void 0, void 0, function* () {
-    const result = yield getKnexDb().count("*", { as: "countAll" }).from("address_point").first();
+    const result = yield getKnexDb()
+        .count("*", { as: "countAll" })
+        .from("address_point")
+        .first();
     return Number(result.countAll) >= 2900000; // total is almost 3 million
 });
 const addressPointSelect = `
@@ -178,10 +181,7 @@ export const checkStreetExists = (streetName, founder) => __awaiter(void 0, void
 });
 const getAllStreets = (cityCode) => __awaiter(void 0, void 0, void 0, function* () {
     const knex = getKnexDb();
-    return yield knex
-        .pluck("name")
-        .from("street")
-        .where("city_code", cityCode);
+    return yield knex.pluck("name").from("street").where("city_code", cityCode);
 });
 export const findAddressPoints = (smdLine, municipality) => __awaiter(void 0, void 0, void 0, function* () {
     const knex = getKnexDb();

package/dist/db/db.d.ts

@@ -21,11 +21,11 @@ export declare const nonEmptyOrNull: (value: string) => string | null;
  * Efficiently insert multiple rows. If preventDuplicatesByFirstColumn is true, the first
  * column should be unique (PK or UNIQUE).
  */
-export declare const insertMultipleRows: (rows: string[][], table: string, columnNames: string[], preventDuplicatesByFirstColumn?: boolean) => Promise<number>;
+export declare const insertMultipleRows: (rows: string[][], table: string, columnNames: string[], preventDuplicates?: boolean, keyColumns?: string[]) => Promise<number>;
 /**
  * Insert a single row and return the autoincremented ID.
  */
-export declare const insertAutoincrementRow: (row: string[], table: string, columnNames: string[]) => Promise<number | null>;
+export declare const insertAutoincrementRow: (row: any[], table: string, columnNames: string[]) => Promise<number | null>;
 export declare const deleteMultipleRowsKnex: (keys: string[], table: string, keyColumnName: string) => Promise<void>;
 export declare const clearDuplicates: (rows: string[][], table: string, columnNames: string[]) => Promise<string[][]>;
 export declare const generatePlaceholders: (n: number) => string;

package/dist/db/db.js

@@ -39,6 +39,12 @@ export const getKnexDb = (config = {}) => {
                 connection: {
                     filename: _knexDbConfig.filePath,
                 },
+                pool: {
+                    afterCreate: (db, done) => {
+                        db.pragma('journal_mode = WAL;');
+                        done(false, done);
+                    },
+                },
                 useNullAsDefault: true,
                 debug: _knexDbConfig.verbose,
             });
@@ -109,7 +115,7 @@ export const nonEmptyOrNull = (value) => {
  * Efficiently insert multiple rows. If preventDuplicatesByFirstColumn is true, the first
  * column should be unique (PK or UNIQUE).
  */
-export const insertMultipleRows = (rows, table, columnNames, preventDuplicatesByFirstColumn = true) => __awaiter(void 0, void 0, void 0, function* () {
+export const insertMultipleRows = (rows, table, columnNames, preventDuplicates = true, keyColumns = []) => __awaiter(void 0, void 0, void 0, function* () {
     if (rows.length === 0) {
         return 0;
     }
@@ -120,8 +126,11 @@ export const insertMultipleRows = (rows, table, columnNames, preventDuplicatesBy
     //   }
     // }
     const insertPlaceholders = generate2DPlaceholders(columnNames.length, rows.length);
-    const onConfict = preventDuplicatesByFirstColumn
-        ? `ON CONFLICT (${columnNames[0]}) DO NOTHING`
+    if (preventDuplicates && keyColumns.length === 0) {
+        keyColumns = [columnNames[0]];
+    }
+    const onConfict = preventDuplicates
+        ? `ON CONFLICT (${keyColumns.join(", ")}) DO NOTHING`
         : "";
     yield getKnexDb().raw(`INSERT INTO ${table} (${columnNames.join(",")}) VALUES ${insertPlaceholders} ${onConfict}`, rows.flat());
     return rows.length;

package/dist/db/founders.js

@@ -85,7 +85,7 @@ export const insertFounders = (founders) => __awaiter(void 0, void 0, void 0, fu
             });
         }
     }
-    const insertedConnections = yield insertMultipleRows(schoolFounderConnectionData, "school_founder", ["school_izo", "founder_id"]);
+    const insertedConnections = yield insertMultipleRows(schoolFounderConnectionData, "school_founder", ["school_izo", "founder_id"], true, ["school_izo", "founder_id"]);
     return insertedFounders + insertedConnections;
 });
 const getCityOfSchool = (izo) => __awaiter(void 0, void 0, void 0, function* () {

package/dist/db/migrations/20230704125715_initial.js

@@ -126,6 +126,7 @@ export function up(knex) {
                 .references("id")
                 .inTable("founder")
                 .notNullable();
+            table_13.unique(["school_izo", "founder_id"]);
         });
         yield knex.schema.createTable("address_point", function (table) {
             table.increments("id");
@@ -172,6 +173,13 @@ export function up(knex) {
                 .onDelete("CASCADE");
             table_14.unique(["school_izo", "address_point_id"]);
         });
+        yield knex.schema.createTable("sync_log", function (table_15) {
+            table_15.increments("id");
+            table_15.text("part").notNullable();
+            table_15.dateTime("started_at").notNullable();
+            table_15.dateTime("finished_at");
+            table_15.boolean("completed").defaultTo(false);
+        });
         yield knex("founder_type").insert([
             { code: 0, name: "Zatím neurčeno" },
             { code: 101, name: "Fyzická osoba" },
@@ -218,16 +226,28 @@ export function up(knex) {
         yield knex.schema.raw("CREATE INDEX municipality_part_code ON address_point (municipality_part_code);");
         yield knex.schema.raw("CREATE INDEX object_type_id ON address_point (object_type_id);");
         yield knex.schema.raw("CREATE INDEX street_code ON address_point (street_code);");
-        yield knex.schema.raw(`CREATE INDEX street_name ON street (name${isSqlite(knex) ? " COLLATE NOCASE" : ""});`);
         yield knex.schema.raw("CREATE INDEX street_sync_feed_url ON street_sync (feed_url);");
         if (isPostgres(knex)) {
             yield knex.schema.raw("CREATE EXTENSION IF NOT EXISTS CITEXT");
-            yield knex.schema.alterTable("street", (t) => {
-                t.specificType("name", "CITEXT").notNullable().alter();
-            });
         }
+        // case insensitive indexes
+        createCaseInsensitiveIndex(knex, "street", "name");
+        createCaseInsensitiveIndex(knex, "city", "name");
+        createCaseInsensitiveIndex(knex, "founder", "name");
+        createCaseInsensitiveIndex(knex, "founder", "short_name");
+        createCaseInsensitiveIndex(knex, "region", "name");
+        createCaseInsensitiveIndex(knex, "county", "name");
+        createCaseInsensitiveIndex(knex, "orp", "name");
     });
 }
+const createCaseInsensitiveIndex = (knex, table, column) => __awaiter(void 0, void 0, void 0, function* () {
+    if (isPostgres(knex)) {
+        yield knex.schema.alterTable(table, (t) => {
+            t.specificType(column, "CITEXT").notNullable().alter();
+        });
+    }
+    yield knex.schema.raw(`CREATE INDEX ${table}_${column} ON ${table} (${column}${isSqlite(knex) ? " COLLATE NOCASE" : ""});`);
+});
 export function down(knex) {
     return __awaiter(this, void 0, void 0, function* () {
         yield knex.schema.dropTableIfExists("school_location");
@@ -247,5 +267,6 @@ export function down(knex) {
         yield knex.schema.dropTableIfExists("prague_district");
         yield knex.schema.dropTableIfExists("school");
         yield knex.schema.dropTableIfExists("street_sync");
+        yield knex.schema.dropTableIfExists("sync_log");
     });
 }

package/dist/db/schools.js

@@ -38,7 +38,7 @@ export const insertSchools = (schools) => __awaiter(void 0, void 0, void 0, func
     // plus filter out duplicate locations (same address id + izo)
     for (const location of locations) {
         try {
-            insertedLocations += yield insertMultipleRows([location], "school_location", ["school_izo", "address_point_id"], false);
+            insertedLocations += yield insertMultipleRows([location], "school_location", ["school_izo", "address_point_id"], true, ["school_izo", "address_point_id"]);
         }
         catch (error) {
             console.log(`Cannot add location with RUIAN code ${location[1]} (school IZO = ${location[0]}): code does not exist.`);

package/dist/db/sync-log.d.ts

@@ -0,0 +1,9 @@
+import { SyncPart } from "./types";
+export declare const startSyncPart: (part: SyncPart) => Promise<number>;
+export declare const setSyncPartAsCompleted: (id: number) => Promise<void>;
+export declare const isSyncPartCompleted: (part: SyncPart) => Promise<boolean>;
+export declare const isEverythingSynced: () => Promise<boolean>;
+export declare const millisecondsSinceLastSyncOfPart: (part: SyncPart) => Promise<number | null>;
+export declare const lastSyncOfPart: (part: SyncPart) => Promise<Date | null>;
+export declare const millisecondsSinceLastSync: () => Promise<number | null>;
+export declare const lastSync: () => Promise<Date | null>;

package/dist/db/sync-log.js

@@ -0,0 +1,74 @@
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import { getKnexDb, insertAutoincrementRow, isSqlite } from "./db";
+import { SyncPart } from "./types";
+const allParts = [
+    SyncPart.AddressPoints,
+    SyncPart.Schools,
+    SyncPart.Regions,
+    SyncPart.Streets,
+];
+const serializeDate = (date) => {
+    return isSqlite(getKnexDb()) ? date.toISOString().slice(0, -1) : date;
+};
+const deserializeDate = (date) => {
+    return isSqlite(getKnexDb()) ? new Date(date) : date;
+};
+export const startSyncPart = (part) => __awaiter(void 0, void 0, void 0, function* () {
+    return yield insertAutoincrementRow([part.toString(), serializeDate(new Date())], "sync_log", ["part", "started_at"]);
+});
+export const setSyncPartAsCompleted = (id) => __awaiter(void 0, void 0, void 0, function* () {
+    yield getKnexDb()
+        .update({ finished_at: serializeDate(new Date()), completed: true })
+        .from("sync_log")
+        .where({ id });
+});
+export const isSyncPartCompleted = (part) => __awaiter(void 0, void 0, void 0, function* () {
+    const result = yield getKnexDb()
+        .select("id")
+        .from("sync_log")
+        .where({ part, completed: true })
+        .limit(1);
+    return result.length > 0;
+});
+export const isEverythingSynced = () => __awaiter(void 0, void 0, void 0, function* () {
+    const results = yield Promise.all(allParts.map(isSyncPartCompleted));
+    return results.every((r) => r);
+});
+export const millisecondsSinceLastSyncOfPart = (part) => __awaiter(void 0, void 0, void 0, function* () {
+    const result = yield lastSyncOfPart(part);
+    return result === null ? null : new Date().getTime() - result.getTime();
+});
+export const lastSyncOfPart = (part) => __awaiter(void 0, void 0, void 0, function* () {
+    const result = yield getKnexDb()
+        .select("finished_at")
+        .from("sync_log")
+        .where({ part, completed: true })
+        .orderBy("finished_at", "desc")
+        .limit(1);
+    if (result.length === 0) {
+        return null;
+    }
+    return deserializeDate(result[0].finished_at);
+});
+export const millisecondsSinceLastSync = () => __awaiter(void 0, void 0, void 0, function* () {
+    const results = yield Promise.all(allParts.map(millisecondsSinceLastSyncOfPart));
+    if (results.some((r) => r === null)) {
+        return null;
+    }
+    return Math.max(...results);
+});
+export const lastSync = () => __awaiter(void 0, void 0, void 0, function* () {
+    const results = yield Promise.all(allParts.map(lastSyncOfPart));
+    if (results.some((r) => r === null)) {
+        return null;
+    }
+    return new Date(Math.min(...results.map((r) => r.getTime())));
+});

package/dist/db/types.d.ts

@@ -38,3 +38,9 @@ export interface DbfStreet {
     NAZEV: string;
     OBEC_KOD: string;
 }
+export declare enum SyncPart {
+    AddressPoints = "address-points",
+    Schools = "schools",
+    Regions = "regions",
+    Streets = "streets"
+}

package/dist/db/types.js

@@ -10,3 +10,10 @@ export const founderToMunicipality = (founder) => {
         code: founder.cityOrDistrictCode,
     };
 };
+export var SyncPart;
+(function (SyncPart) {
+    SyncPart["AddressPoints"] = "address-points";
+    SyncPart["Schools"] = "schools";
+    SyncPart["Regions"] = "regions";
+    SyncPart["Streets"] = "streets";
+})(SyncPart || (SyncPart = {}));

package/dist/open-data-sync/address-points.js

@@ -15,9 +15,10 @@ import fetch from "node-fetch";
 import { join } from "path";
 import { pipeline } from "stream/promises";
 import { commitAddressPoints } from "../db/address-points";
-import { disconnectKnex } from "../db/db";
+import { SyncPart } from "../db/types";
 import { getLatestUrlFromAtomFeed } from "../utils/atom";
 import { prepareOptions, } from "../utils/helpers";
+import { runSyncPart } from "./common";
 const maxBufferSize = 1000;
 const downloadAndUnzip = (url, options) => __awaiter(void 0, void 0, void 0, function* () {
     const zipFilePath = join(options.tmpDir, options.addressPointsZipFileName);
@@ -63,17 +64,11 @@ const importDataToDb = (options) => __awaiter(void 0, void 0, void 0, function*
 });
 const getExtractionFolder = (options) => join(options.tmpDir, options.addressPointsCsvFolderName);
 export const downloadAndImportAddressPoints = (options = {}) => __awaiter(void 0, void 0, void 0, function* () {
-    try {
+    yield runSyncPart(SyncPart.AddressPoints, [], () => __awaiter(void 0, void 0, void 0, function* () {
         const completeOptions = prepareOptions(options);
         const datasetFeedLink = yield getLatestUrlFromAtomFeed(completeOptions.addressPointsAtomUrl);
         const zipUrl = yield getLatestUrlFromAtomFeed(datasetFeedLink);
-        // await downloadAndUnzip(zipUrl, completeOptions);
+        yield downloadAndUnzip(zipUrl, completeOptions);
         yield importDataToDb(completeOptions);
-    }
-    catch (error) {
-        console.error(error);
-    }
-    finally {
-        yield disconnectKnex();
-    }
+    }));
 });

package/dist/open-data-sync/common.d.ts

@@ -0,0 +1,2 @@
+import { SyncPart } from "../db/types";
+export declare const runSyncPart: (part: SyncPart, dependencies: SyncPart[], partFunction: () => Promise<void>) => Promise<void>;

package/dist/open-data-sync/common.js

@@ -0,0 +1,30 @@
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import { disconnectKnex } from "../db/db";
+import { isSyncPartCompleted, setSyncPartAsCompleted, startSyncPart } from "../db/sync-log";
+export const runSyncPart = (part, dependencies, partFunction) => __awaiter(void 0, void 0, void 0, function* () {
+    try {
+        const logId = yield startSyncPart(part);
+        for (const dependency of dependencies) {
+            const isCompleted = yield isSyncPartCompleted(dependency);
+            if (!isCompleted) {
+                throw new Error(`Sync part ${part} cannot be started because dependency ${dependency} is not completed. Run 'npm run ${dependency}' first.`);
+            }
+        }
+        yield partFunction();
+        yield setSyncPartAsCompleted(logId);
+    }
+    catch (error) {
+        console.log(error);
+    }
+    finally {
+        yield disconnectKnex();
+    }
+});

package/dist/open-data-sync/regions.js

@@ -12,8 +12,9 @@ import iconv from "iconv-lite";
 import fetch from "node-fetch";
 import { pipeline } from "stream/promises";
 import { insertRegionsAndOrps } from "../db/regions";
+import { SyncPart } from "../db/types";
 import { prepareOptions } from "../utils/helpers";
-import { disconnectKnex } from "../db/db";
+import { runSyncPart } from "./common";
 const downloadAndImportDataToDb = (options) => __awaiter(void 0, void 0, void 0, function* () {
     console.log("Downloading regions and ORP data...");
     const response = yield fetch(options.regionsCsvUrl);
@@ -37,14 +38,8 @@ const downloadAndImportDataToDb = (options) => __awaiter(void 0, void 0, void 0,
     console.log(`Import completed. Total imported rows: ${rows.length}`);
 });
 export const downloadAndImportRegions = (options = {}) => __awaiter(void 0, void 0, void 0, function* () {
-    try {
+    yield runSyncPart(SyncPart.Regions, [], () => __awaiter(void 0, void 0, void 0, function* () {
         const completeOptions = prepareOptions(options);
         yield downloadAndImportDataToDb(completeOptions);
-    }
-    catch (error) {
-        console.log(error);
-    }
-    finally {
-        yield disconnectKnex();
-    }
+    }));
 });

package/dist/open-data-sync/schools.js

@@ -12,11 +12,11 @@ import fetch from "node-fetch";
 import { join } from "path";
 import sax from "sax";
 import { pipeline } from "stream/promises";
-import { disconnectKnex } from "../db/db";
 import { insertFounders } from "../db/founders";
 import { insertSchools } from "../db/schools";
-import { MunicipalityType } from "../db/types";
+import { MunicipalityType, SyncPart } from "../db/types";
 import { prepareOptions, } from "../utils/helpers";
+import { runSyncPart } from "./common";
 const downloadXml = (options) => __awaiter(void 0, void 0, void 0, function* () {
     if (existsSync(getXmlFilePath(options))) {
         return;
@@ -300,18 +300,12 @@ const getXmlFilePath = (options) => {
     return join(options.tmpDir, options.schoolsXmlFileName);
 };
 export const downloadAndImportSchools = (options, saveFoundersToCsv = false, saveSchoolsWithoutRuianToCsv = false) => __awaiter(void 0, void 0, void 0, function* () {
-    try {
+    yield runSyncPart(SyncPart.Schools, [SyncPart.AddressPoints], () => __awaiter(void 0, void 0, void 0, function* () {
         const runOptions = prepareOptions(options);
         yield downloadXml(runOptions);
         yield importDataToDb(runOptions, saveFoundersToCsv, saveSchoolsWithoutRuianToCsv);
         deleteSchoolsXmlFile(runOptions);
-    }
-    catch (error) {
-        console.error(error);
-    }
-    finally {
-        yield disconnectKnex();
-    }
+    }));
 });
 export const deleteSchoolsXmlFile = (options = {}) => {
     const runOptions = prepareOptions(options);

package/dist/open-data-sync/streets.js

@@ -14,10 +14,10 @@ import parseDBF from "parsedbf";
 import { join } from "path";
 import { pipeline } from "stream/promises";
 import { deleteStreets, getAllSyncedStreets, insertStreetsFromDbf, setStreetAsSynced, } from "../db/street-sync";
+import { SyncPart } from "../db/types";
 import { getAllUrlsFromAtomFeed, getLatestUrlFromAtomFeed, } from "../utils/atom";
 import { prepareOptions, } from "../utils/helpers";
-import { disconnectKnex } from "../db/db";
-import { areAddressPointsSynced } from "../db/address-points";
+import { runSyncPart } from "./common";
 const prepareFolders = (options) => {
     const tempFolder = getTempFolder(options);
     if (!existsSync(tempFolder)) {
@@ -48,11 +48,7 @@ const importDataToDb = (data) => __awaiter(void 0, void 0, void 0, function* ()
     yield insertStreetsFromDbf(data);
 });
 export const downloadAndImportStreets = (options) => __awaiter(void 0, void 0, void 0, function* () {
-    try {
-        // first check if address points are already synced
-        if (!(yield areAddressPointsSynced())) {
-            throw new Error("Address points are not synced yet. First run 'npm run address-points' and then try again.");
-        }
+    yield runSyncPart(SyncPart.Streets, [SyncPart.AddressPoints], () => __awaiter(void 0, void 0, void 0, function* () {
         console.log("Starting to download and import streets. This takes up to 1 hour.");
         const completeOptions = prepareOptions(options);
         prepareFolders(completeOptions);
@@ -81,11 +77,5 @@ export const downloadAndImportStreets = (options) => __awaiter(void 0, void 0, v
             done++;
             console.log(`Loaded links: ${done}/${allDatasetFeedLinks.length}`);
         }
-    }
-    catch (error) {
-        console.log(error);
-    }
-    finally {
-        yield disconnectKnex();
-    }
+    }));
 });

package/package.json

@@ -39,6 +39,7 @@
     "streets": "ts-node --experimental-specifier-resolution=node --esm src/bin/streets.ts",
     "diagrams": "ts-node --experimental-specifier-resolution=node --esm src/bin/generate-syntax-diagrams.ts",
     "clear-all": "ts-node --experimental-specifier-resolution=node --esm src/bin/clear-all.ts",
+    "init-db": "ts-node --experimental-specifier-resolution=node --esm src/bin/init-db.ts",
     "parse-ordinance": "ts-node --experimental-specifier-resolution=node --esm src/bin/parse-ordinance.ts",
     "knex": "ts-node --experimental-specifier-resolution=node --esm src/bin/knex.ts",
     "test": "node --experimental-vm-modules --no-warnings node_modules/jest/bin/jest.js -c=jest-esm.config.mjs --no-cache",
@@ -52,7 +53,7 @@
   },
   "name": "text-to-map",
   "description": "Text To Map usiluje o lepší, strojově zpracovatelné využití částí vyhlášek s výčtem ulic a dalších lokací. Jde o rozšiřitelnou sadu konceptů a nástrojů, které zajistí hladký převod výčtu ulic a jejich rozsahů v lidsky srozumitelném jazyce do strojově zpracovatelného, uchopitelného formátu.",
-  "version": "1.1.3",
+  "version": "1.1.5",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "files": [