text-to-map 1.0.0

package/dist/db/founders.js

@@ -0,0 +1,191 @@
+import { MunicipalityType } from "./types";
+import { getDb, insertAutoincrementRow, insertMultipleRows } from "./db";
+import { extractMunicipalityName, findClosestString } from "../utils/helpers";
+const cityTypeCode = 261;
+const cityDistrictTypeCode = 263;
+export const insertFounders = (founders) => {
+    const db = getDb();
+    const selectCityStatement = db.prepare(`SELECT c.name, c.code FROM school s
+    JOIN school_location l ON s.izo = l.school_izo
+    JOIN address_point a ON l.address_point_id = a.id
+    JOIN city c ON a.city_code = c.code
+    WHERE s.izo = ?
+    LIMIT 1`);
+    const selectDistrictStatement = db.prepare(`SELECT d.name, d.code FROM school s
+    JOIN school_location l ON s.izo = l.school_izo
+    JOIN address_point a ON l.address_point_id = a.id
+    JOIN city_district d ON a.city_district_code = d.code
+    WHERE s.izo = ?
+    LIMIT 1`);
+    let insertedFounders = 0;
+    const schoolFounderConnectionData = [];
+    founders.forEach((founder) => {
+        if (founder.municipalityType !== MunicipalityType.City &&
+            founder.municipalityType !== MunicipalityType.District) {
+            return;
+        }
+        const municipalityName = extractMunicipalityName(founder);
+        // check if the extracted municipality name is the same as in all the schools' locations
+        let allEqual = true;
+        let municipalityCode = "";
+        founder.schools.forEach((school) => {
+            const result = (founder.municipalityType === MunicipalityType.City
+                ? selectCityStatement
+                : selectDistrictStatement).get(school.izo);
+            if (!result) {
+                console.log(`izo: ${school.izo}, extracted: ${municipalityName}, RUIAN: UNDEFINED`);
+                allEqual = false;
+            }
+            else {
+                const { name, code } = result;
+                municipalityCode = code;
+                if (name !== municipalityName) {
+                    console.log(`izo: ${school.izo}, extracted: ${municipalityName}, RUIAN: ${name}`);
+                    allEqual = false;
+                }
+            }
+        });
+        if (!allEqual) {
+            // here we could try to find the correct municipality by searching for the name in the DB
+            // this would solve the case of one school in the whole Czech Republic that is located outside
+            // its founder municipality - too much work for now
+            return;
+        }
+        const founderId = insertAutoincrementRow([
+            founder.name,
+            founder.ico,
+            String(founder.originalType),
+            founder.municipalityType === MunicipalityType.City
+                ? municipalityCode
+                : null,
+            founder.municipalityType === MunicipalityType.City
+                ? null
+                : municipalityCode,
+        ], "founder", ["name", "ico", "founder_type_code", "city_code", "city_district_code"]);
+        if (founderId !== null) {
+            insertedFounders++;
+            founder.schools.forEach((school) => {
+                schoolFounderConnectionData.push([school.izo, founderId]);
+            });
+        }
+    });
+    const insertedConnections = insertMultipleRows(schoolFounderConnectionData, "school_founder", ["school_izo", "founder_id"]);
+    return insertedFounders + insertedConnections;
+};
+export const findFounder = (name) => {
+    const errors = [];
+    const db = getDb();
+    const exactMatchStatement = db.prepare(`SELECT f.id, f.name, f.ico, f.founder_type_code, f.city_code, f.city_district_code FROM founder f
+    LEFT JOIN city c ON c.code = f.city_code
+    LEFT JOIN city_district d ON d.code = f.city_district_code
+    WHERE f.name = ? OR c.name = ? OR d.name = ?`);
+    const result = exactMatchStatement.get(name, name, name);
+    if (result) {
+        return { founder: resultToFounder(result), errors };
+    }
+    else {
+        const allFounderNames = getAllFounderNames();
+        const namesList = allFounderNames
+            .map((row) => row.founderName)
+            .concat(allFounderNames.map((row) => row.municipalityName));
+        const bestMatch = findClosestString(name, namesList);
+        const bestMatchRow = allFounderNames.find((foundersNames) => foundersNames.founderName === bestMatch ||
+            foundersNames.municipalityName === bestMatch);
+        errors.push(`No exact match for founder "${name}", using "${bestMatch}" instead.`);
+        const findByIdStatement = db.prepare(`SELECT f.id, f.name, f.ico, f.founder_type_code, f.city_code, f.city_district_code FROM founder f
+      WHERE f.id = ?`);
+        return {
+            founder: resultToFounder(findByIdStatement.get(bestMatchRow.id)),
+            errors,
+        };
+    }
+};
+let cachedCityCode = null;
+let cityCodeFounder = null;
+export const getFounderCityCode = (founder) => {
+    const db = getDb();
+    if (founder.municipalityType === MunicipalityType.District) {
+        if (cityCodeFounder !== founder) {
+            const founderStatement = db.prepare(`
+        SELECT city_code
+        FROM city_district
+        WHERE code = ?
+      `);
+            cachedCityCode = founderStatement.get(founder.cityOrDistrictCode).city_code;
+        }
+        return cachedCityCode;
+    }
+    else {
+        return founder.cityOrDistrictCode;
+    }
+};
+const resultToFounder = (result) => {
+    return {
+        name: result.name,
+        ico: result.ico,
+        originalType: result.founder_type_code,
+        municipalityType: result.founder_type_code === cityTypeCode
+            ? MunicipalityType.City
+            : MunicipalityType.District,
+        cityOrDistrictCode: result.city_code || result.city_district_code,
+        schools: getSchoolsByFounderId(parseInt(result.id)),
+    };
+};
+const getSchoolsByFounderId = (founderId) => {
+    const db = getDb();
+    const statement = db.prepare(`SELECT s.izo, s.name, s.capacity, sl.address_point_id FROM school s
+    JOIN school_founder sf ON s.izo = sf.school_izo
+    JOIN school_location sl ON s.izo = sl.school_izo
+    WHERE sf.founder_id = ?`);
+    return statement.all(founderId).map((row) => ({
+        izo: String(row.izo),
+        name: String(row.name),
+        capacity: parseInt(row.capacity),
+        locations: [
+            {
+                addressPointId: parseInt(row.address_point_id),
+            },
+        ],
+    }));
+};
+const getAllFounderNames = () => {
+    const db = getDb();
+    const statement = db.prepare(`SELECT f.id, f.name AS founder_name, c.name AS city_name, d.name AS city_district_name FROM founder f
+    LEFT JOIN city c ON c.code = f.city_code
+    LEFT JOIN city_district d ON d.code = f.city_district_code`);
+    const result = statement.all();
+    return result.map((row) => ({
+        id: parseInt(row.id),
+        founderName: String(row.founder_name),
+        municipalityName: String(row.city_name ? row.city_name : row.city_district_name),
+    }));
+};
+export const findMunicipalityByNameAndType = (name, type) => {
+    const errors = [];
+    const db = getDb();
+    const exactMatchStatement = db.prepare(`SELECT code FROM ${type === MunicipalityType.City ? "city" : "city_district"} WHERE name = ?`);
+    const result = exactMatchStatement.get(name);
+    if (result) {
+        return { municipality: { code: result.code, type }, errors };
+    }
+    else {
+        const allNames = getAllMunicipalityNames(type);
+        const namesList = allNames.map((row) => row.name);
+        const bestMatch = findClosestString(name, namesList);
+        const bestMatchRow = allNames.find((municipality) => municipality.name === bestMatch);
+        errors.push(`No exact match for municipality "${name}", using "${bestMatch}" instead.`);
+        return {
+            municipality: { code: bestMatchRow.code, type },
+            errors,
+        };
+    }
+};
+const getAllMunicipalityNames = (type) => {
+    const db = getDb();
+    const statement = db.prepare(`SELECT name, code FROM ${type === MunicipalityType.City ? "city" : "city_district"}`);
+    const result = statement.all();
+    return result.map((row) => ({
+        name: row.name,
+        code: row.code,
+    }));
+};

package/dist/db/meta.d.ts

@@ -0,0 +1,3 @@
+/// <reference types="better-sqlite3" />
+export declare const getMetaValue: (key: string) => string | undefined;
+export declare const setMetaValue: (key: string, value: any) => import("better-sqlite3").RunResult;

package/dist/db/meta.js

@@ -0,0 +1,17 @@
+import { getDb } from "./db";
+export const getMetaValue = (key) => {
+    const db = getDb();
+    const statement = db.prepare("SELECT value FROM meta WHERE key = ?");
+    return statement.pluck().get(key);
+};
+export const setMetaValue = (key, value) => {
+    const db = getDb();
+    if (getMetaValue(key) === undefined) {
+        const statement = db.prepare("INSERT INTO meta (key, value) VALUES (?, ?)");
+        return statement.run(key, value.toString());
+    }
+    else {
+        const statement = db.prepare("UPDATE meta SET value = ? WHERE key = ?");
+        return statement.run(value.toString(), key);
+    }
+};

package/dist/db/schools.d.ts

@@ -0,0 +1,6 @@
+import { School } from "./types";
+export declare const insertSchools: (schools: School[], doNotClearTable?: boolean) => number;
+export declare const findSchool: (name: string, schools: School[]) => {
+    school: School | null;
+    errors: string[];
+};

package/dist/db/schools.js

@@ -0,0 +1,61 @@
+import { getDb, insertMultipleRows } from "./db";
+import { findClosestString } from "../utils/helpers";
+export const insertSchools = (schools, doNotClearTable = false) => {
+    if (!doNotClearTable) {
+        const db = getDb();
+        db.exec("DELETE FROM school_location");
+        db.exec("DELETE FROM school_founder");
+        db.exec("DELETE FROM school_location");
+        db.exec("DELETE FROM founder");
+        db.exec("DELETE FROM school");
+        db.exec("DELETE FROM sqlite_sequence WHERE name='school_location' OR name='founder' OR name='school_founder'");
+    }
+    const insertedSchools = insertMultipleRows(schools.map((school) => [
+        school.izo,
+        school.name,
+        school.capacity.toString(),
+    ]), "school", ["izo", "name", "capacity"]);
+    // const locations = schools.flatMap((school) => {
+    //   const uniqueAddressPoints = [
+    //     ...new Set(school.locations.map((location) => location.addressPointId)),
+    //   ];
+    //   return uniqueAddressPoints.map((addressPoint) => [
+    //     school.izo,
+    //     addressPoint.toString(),
+    //   ]);
+    // });
+    const locations = schools
+        .filter((school) => school.locations.length > 0)
+        .map((school) => [
+        school.izo,
+        school.locations[0].addressPointId.toString(),
+    ]);
+    let insertedLocations = 0;
+    // @todo prevent FK errors (address point might not exist, maybe even school?) - or maybe the DB wasn't fresh
+    // plus filter out duplicit locations (same address id + izo)
+    locations.forEach((location) => {
+        try {
+            insertedLocations += insertMultipleRows([location], "school_location", ["school_izo", "address_point_id"], false);
+        }
+        catch (error) {
+            console.log(`Cannot add location with RUIAN code ${location[1]} (school IZO = ${location[0]}): code does not exist.`);
+        }
+    });
+    return insertedSchools + insertedLocations;
+};
+export const findSchool = (name, schools) => {
+    if (schools.length === 0) {
+        return { school: null, errors: ["No schools for this founder."] };
+    }
+    let school = null;
+    const errors = [];
+    school = schools.find((school) => school.name === name);
+    if (school) {
+        return { school, errors };
+    }
+    const namesList = schools.map((school) => school.name);
+    const bestMatch = findClosestString(name, namesList);
+    school = schools.find((school) => bestMatch === school.name);
+    errors.push(`No exact match for school "${name}", using "${bestMatch}" instead.`);
+    return { school, errors };
+};

package/dist/db/street-sync.d.ts

@@ -0,0 +1,5 @@
+import { DbfStreet } from "./types";
+export declare const setStreetAsSynced: (streetFeedUrl: string) => void;
+export declare const getAllSyncedStreets: () => Set<string>;
+export declare const deleteStreets: (streetUrls: string[]) => void;
+export declare const insertStreetsFromDbf: (data: DbfStreet[]) => number;

package/dist/db/street-sync.js

@@ -0,0 +1,18 @@
+import { deleteMultipleRows, getDb, insertMultipleRows } from "./db";
+export const setStreetAsSynced = (streetFeedUrl) => {
+    const db = getDb();
+    db.prepare("INSERT OR IGNORE INTO street_sync (feed_url) VALUES (?)").run(streetFeedUrl);
+};
+export const getAllSyncedStreets = () => {
+    const db = getDb();
+    return new Set(db
+        .prepare("SELECT * FROM street_sync")
+        .all()
+        .map((row) => row.feed_url));
+};
+export const deleteStreets = (streetUrls) => {
+    deleteMultipleRows(streetUrls, "street_sync", "feed_url");
+};
+export const insertStreetsFromDbf = (data) => {
+    return insertMultipleRows(data.map((street) => [street.KOD, street.OBEC_KOD, street.NAZEV]), "street", ["code", "city_code", "name"]);
+};

package/dist/db/types.d.ts

@@ -0,0 +1,34 @@
+export interface SchoolLocation {
+    id?: number;
+    schoolIzo?: number;
+    addressPointId: number;
+}
+export interface School {
+    name: string;
+    izo: string;
+    capacity: number;
+    locations: SchoolLocation[];
+}
+export interface Founder {
+    name: string;
+    ico: string;
+    originalType?: number;
+    municipalityType: MunicipalityType;
+    cityOrDistrictCode?: number;
+    schools: School[];
+}
+export declare enum MunicipalityType {
+    City = 0,
+    District = 1,
+    Other = 2
+}
+export interface Municipality {
+    type: MunicipalityType;
+    code: number;
+}
+export declare const founderToMunicipality: (founder: Founder) => Municipality;
+export interface DbfStreet {
+    KOD: string;
+    NAZEV: string;
+    OBEC_KOD: string;
+}

package/dist/db/types.js

@@ -0,0 +1,12 @@
+export var MunicipalityType;
+(function (MunicipalityType) {
+    MunicipalityType[MunicipalityType["City"] = 0] = "City";
+    MunicipalityType[MunicipalityType["District"] = 1] = "District";
+    MunicipalityType[MunicipalityType["Other"] = 2] = "Other";
+})(MunicipalityType || (MunicipalityType = {}));
+export const founderToMunicipality = (founder) => {
+    return {
+        type: founder.municipalityType,
+        code: founder.cityOrDistrictCode,
+    };
+};

package/dist/index.d.ts

@@ -0,0 +1,4 @@
+export { downloadAndImportAllLatestAddressPoints, importAllLatestAddressPoints, deleteDb } from "./open-data-sync/address-points";
+export { downloadAndImportAllSchools, deleteSchoolsXmlFile } from "./open-data-sync/schools";
+export { downloadAndImportAllStreets } from "./open-data-sync/streets";
+export { parseOrdinanceToAddressPoints } from "./street-markdown/smd";

package/dist/index.js

@@ -0,0 +1,4 @@
+export { downloadAndImportAllLatestAddressPoints, importAllLatestAddressPoints, deleteDb } from "./open-data-sync/address-points";
+export { downloadAndImportAllSchools, deleteSchoolsXmlFile } from "./open-data-sync/schools";
+export { downloadAndImportAllStreets } from "./open-data-sync/streets";
+export { parseOrdinanceToAddressPoints } from "./street-markdown/smd";

package/dist/open-data-sync/address-points.d.ts

@@ -0,0 +1,4 @@
+import { OpenDataSyncOptions } from "../utils/helpers";
+export declare const downloadAndImportAllLatestAddressPoints: (options?: OpenDataSyncOptions) => Promise<void>;
+export declare const importAllLatestAddressPoints: (options?: OpenDataSyncOptions) => Promise<void>;
+export declare const deleteDb: (options?: OpenDataSyncOptions) => void;

package/dist/open-data-sync/address-points.js

@@ -0,0 +1,73 @@
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import fetch from "node-fetch";
+import { createWriteStream, createReadStream, rmSync, readdirSync, existsSync, } from "fs";
+import { pipeline } from "stream/promises";
+import { join } from "path";
+import AdmZip from "adm-zip";
+import { parse } from "csv-parse";
+import iconv from "iconv-lite";
+import { prepareOptions, initDb, } from "../utils/helpers";
+import { commitAddressPoints, importParsedLine } from "../db/address-points";
+import { getLatestUrlFromAtomFeed } from "../utils/atom";
+const downloadAndUnzip = (url, options) => __awaiter(void 0, void 0, void 0, function* () {
+    const zipFilePath = join(options.tmpDir, options.addressPointsZipFileName);
+    console.log("Downloading a large ZIP file with RUIAN data (~65 MB)...");
+    const response = yield fetch(url);
+    if (response.status !== 200) {
+        throw new Error(`The ZIP file could not be downloaded. HTTP Code: ${response.status}`);
+    }
+    yield pipeline(response.body, createWriteStream(zipFilePath));
+    console.log("Finished downloading.");
+    const zip = new AdmZip(zipFilePath);
+    console.log(`Starting to unzip CSV files to '${options.tmpDir}'`);
+    zip.extractAllTo(options.tmpDir, true);
+    console.log("Unzip completed.");
+    rmSync(zipFilePath);
+    console.log("Removed the ZIP file.");
+});
+const importDataToDb = (options) => __awaiter(void 0, void 0, void 0, function* () {
+    const extractionFolder = getExtractionFolder(options);
+    const files = readdirSync(extractionFolder);
+    let total = 0;
+    let next = 0;
+    console.log("Initiating import of RUIAN data to search DB (~3 million rows to be imported).");
+    initDb(options);
+    for (const file of files) {
+        const parseStream = parse({ delimiter: ";", fromLine: 2 }).on("data", (data) => {
+            total += importParsedLine(data);
+            if (total - next >= 100000) {
+                next += 100000;
+                console.log(`Total imported rows: ${next}`);
+            }
+        });
+        yield pipeline(createReadStream(join(extractionFolder, file)), iconv.decodeStream("win1250"), parseStream);
+        total += commitAddressPoints();
+    }
+    console.log(`Import completed. Total imported rows: ${total}`);
+});
+const getExtractionFolder = (options) => join(options.tmpDir, options.addressPointsCsvFolderName);
+export const downloadAndImportAllLatestAddressPoints = (options = {}) => __awaiter(void 0, void 0, void 0, function* () {
+    const completeOptions = prepareOptions(options);
+    const datasetFeedLink = yield getLatestUrlFromAtomFeed(completeOptions.addressPointsAtomUrl);
+    const zipUrl = yield getLatestUrlFromAtomFeed(datasetFeedLink);
+    yield downloadAndUnzip(zipUrl, completeOptions);
+    yield importDataToDb(completeOptions);
+});
+export const importAllLatestAddressPoints = (options = {}) => __awaiter(void 0, void 0, void 0, function* () {
+    const completeOptions = prepareOptions(options);
+    yield importDataToDb(completeOptions);
+});
+export const deleteDb = (options = {}) => {
+    const completeOptions = prepareOptions(options);
+    if (existsSync(completeOptions.dbFilePath)) {
+        rmSync(completeOptions.dbFilePath);
+    }
+};

package/dist/open-data-sync/schools.d.ts

@@ -0,0 +1,3 @@
+import { OpenDataSyncOptions } from "../utils/helpers";
+export declare const downloadAndImportAllSchools: (options: OpenDataSyncOptions) => Promise<void>;
+export declare const deleteSchoolsXmlFile: (options?: OpenDataSyncOptions) => void;