text-to-map 1.1.2 → 1.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/db/address-points.js +8 -8
- package/dist/db/db.d.ts +3 -3
- package/dist/db/db.js +13 -13
- package/dist/db/migrations/20230704125715_initial.js +25 -5
- package/dist/db/sync-log.d.ts +9 -0
- package/dist/db/sync-log.js +74 -0
- package/dist/db/types.d.ts +6 -0
- package/dist/db/types.js +7 -0
- package/dist/open-data-sync/address-points.js +5 -10
- package/dist/open-data-sync/common.d.ts +2 -0
- package/dist/open-data-sync/common.js +30 -0
- package/dist/open-data-sync/regions.js +4 -9
- package/dist/open-data-sync/schools.js +4 -10
- package/dist/open-data-sync/streets.js +4 -14
- package/package.json +2 -1
|
@@ -11,7 +11,7 @@ import { AddressPointType, createSingleLineAddress } from "czech-address";
|
|
|
11
11
|
import { SeriesType, isNegativeSeriesSpec, isRange, isSeriesSpecArray, isWholeMunicipalitySmdLine, } from "../street-markdown/types";
|
|
12
12
|
import { findClosestString } from "../utils/helpers";
|
|
13
13
|
import jtsk2wgs84 from "../utils/jtsk2wgs84";
|
|
14
|
-
import { extractKeyValuesPairs, generate2DPlaceholders, getKnexDb, insertMultipleRows, isSqlite, nonEmptyOrNull } from "./db";
|
|
14
|
+
import { extractKeyValuesPairs, generate2DPlaceholders, getKnexDb, insertMultipleRows, isSqlite, nonEmptyOrNull, } from "./db";
|
|
15
15
|
import { getFounderCityCode } from "./founders";
|
|
16
16
|
import { MunicipalityType } from "./types";
|
|
17
17
|
const DescriptiveType = "č.p.";
|
|
@@ -108,7 +108,10 @@ export const insertStreets = (buffer) => __awaiter(void 0, void 0, void 0, funct
|
|
|
108
108
|
]), "street", ["code", "city_code", "name"]);
|
|
109
109
|
});
|
|
110
110
|
export const areAddressPointsSynced = () => __awaiter(void 0, void 0, void 0, function* () {
|
|
111
|
-
const result = yield getKnexDb()
|
|
111
|
+
const result = yield getKnexDb()
|
|
112
|
+
.count("*", { as: "countAll" })
|
|
113
|
+
.from("address_point")
|
|
114
|
+
.first();
|
|
112
115
|
return Number(result.countAll) >= 2900000; // total is almost 3 million
|
|
113
116
|
});
|
|
114
117
|
const addressPointSelect = `
|
|
@@ -141,7 +144,7 @@ export const checkStreetExists = (streetName, founder) => __awaiter(void 0, void
|
|
|
141
144
|
const cityCode = yield getFounderCityCode(founder);
|
|
142
145
|
const rowList = yield knex.raw(`SELECT name AS street_name
|
|
143
146
|
FROM street
|
|
144
|
-
WHERE city_code = ? AND name = ? ${isSqlite() ? "COLLATE NOCASE" : ""}`, [cityCode, streetName]);
|
|
147
|
+
WHERE city_code = ? AND name = ? ${isSqlite(knex) ? "COLLATE NOCASE" : ""}`, [cityCode, streetName]);
|
|
145
148
|
if (rowList.length > 0) {
|
|
146
149
|
const row = rowList[0];
|
|
147
150
|
if (row.street_name !== streetName) {
|
|
@@ -178,10 +181,7 @@ export const checkStreetExists = (streetName, founder) => __awaiter(void 0, void
|
|
|
178
181
|
});
|
|
179
182
|
const getAllStreets = (cityCode) => __awaiter(void 0, void 0, void 0, function* () {
|
|
180
183
|
const knex = getKnexDb();
|
|
181
|
-
return yield knex
|
|
182
|
-
.pluck("name")
|
|
183
|
-
.from("street")
|
|
184
|
-
.where("city_code", cityCode);
|
|
184
|
+
return yield knex.pluck("name").from("street").where("city_code", cityCode);
|
|
185
185
|
});
|
|
186
186
|
export const findAddressPoints = (smdLine, municipality) => __awaiter(void 0, void 0, void 0, function* () {
|
|
187
187
|
const knex = getKnexDb();
|
|
@@ -190,7 +190,7 @@ export const findAddressPoints = (smdLine, municipality) => __awaiter(void 0, vo
|
|
|
190
190
|
: [smdLine.street, municipality.code];
|
|
191
191
|
const streetJoinCondition = isWholeMunicipalitySmdLine(smdLine)
|
|
192
192
|
? "LEFT JOIN street s ON a.street_code = s.code"
|
|
193
|
-
: `JOIN street s ON a.street_code = s.code AND s.name = ? ${isSqlite() ? "COLLATE NOCASE" : ""}`;
|
|
193
|
+
: `JOIN street s ON a.street_code = s.code AND s.name = ? ${isSqlite(knex) ? "COLLATE NOCASE" : ""}`;
|
|
194
194
|
const queryResult = yield knex.raw(`SELECT a.id, s.name AS street_name, o.name AS object_type_name, a.descriptive_number,
|
|
195
195
|
a.orientational_number, a.orientational_number_letter, c.name AS city_name,
|
|
196
196
|
d.name AS district_name, m.name AS municipality_part_name, p.name AS prague_district_name,
|
package/dist/db/db.d.ts
CHANGED
|
@@ -11,8 +11,8 @@ export declare enum SupportedDbType {
|
|
|
11
11
|
postgres = "postgres"
|
|
12
12
|
}
|
|
13
13
|
export declare const getKnexDb: (config?: Partial<DbConfig>) => Knex;
|
|
14
|
-
export declare const isPostgres: () => boolean;
|
|
15
|
-
export declare const isSqlite: () => boolean;
|
|
14
|
+
export declare const isPostgres: (knex: Knex) => boolean;
|
|
15
|
+
export declare const isSqlite: (knex: Knex) => boolean;
|
|
16
16
|
export declare const initDb: (config?: Partial<DbConfig>) => Promise<Knex>;
|
|
17
17
|
export declare const clearDb: (config?: Partial<DbConfig>) => Promise<Knex>;
|
|
18
18
|
export declare const disconnectKnex: () => Promise<void>;
|
|
@@ -25,7 +25,7 @@ export declare const insertMultipleRows: (rows: string[][], table: string, colum
|
|
|
25
25
|
/**
|
|
26
26
|
* Insert a single row and return the autoincremented ID.
|
|
27
27
|
*/
|
|
28
|
-
export declare const insertAutoincrementRow: (row:
|
|
28
|
+
export declare const insertAutoincrementRow: (row: any[], table: string, columnNames: string[]) => Promise<number | null>;
|
|
29
29
|
export declare const deleteMultipleRowsKnex: (keys: string[], table: string, keyColumnName: string) => Promise<void>;
|
|
30
30
|
export declare const clearDuplicates: (rows: string[][], table: string, columnNames: string[]) => Promise<string[][]>;
|
|
31
31
|
export declare const generatePlaceholders: (n: number) => string;
|
package/dist/db/db.js
CHANGED
|
@@ -18,6 +18,10 @@ export var SupportedDbType;
|
|
|
18
18
|
SupportedDbType["sqlite"] = "sqlite";
|
|
19
19
|
SupportedDbType["postgres"] = "postgres";
|
|
20
20
|
})(SupportedDbType || (SupportedDbType = {}));
|
|
21
|
+
const DbClient = {
|
|
22
|
+
[SupportedDbType.sqlite]: "better-sqlite3",
|
|
23
|
+
[SupportedDbType.postgres]: "pg",
|
|
24
|
+
};
|
|
21
25
|
const defaults = {
|
|
22
26
|
dbType: SupportedDbType.sqlite,
|
|
23
27
|
filePath: "address_points.db",
|
|
@@ -31,7 +35,7 @@ export const getKnexDb = (config = {}) => {
|
|
|
31
35
|
_knexDbConfig = Object.assign(Object.assign(Object.assign({}, defaults), getEnvConfig()), config);
|
|
32
36
|
if (_knexDbConfig.dbType === SupportedDbType.sqlite) {
|
|
33
37
|
_knexDb = knex({
|
|
34
|
-
client:
|
|
38
|
+
client: DbClient[SupportedDbType.sqlite],
|
|
35
39
|
connection: {
|
|
36
40
|
filename: _knexDbConfig.filePath,
|
|
37
41
|
},
|
|
@@ -41,7 +45,7 @@ export const getKnexDb = (config = {}) => {
|
|
|
41
45
|
}
|
|
42
46
|
else if (_knexDbConfig.dbType === SupportedDbType.postgres) {
|
|
43
47
|
_knexDb = knex({
|
|
44
|
-
client:
|
|
48
|
+
client: DbClient[SupportedDbType.postgres],
|
|
45
49
|
connection: _knexDbConfig.pgConnectionString,
|
|
46
50
|
useNullAsDefault: true,
|
|
47
51
|
});
|
|
@@ -52,17 +56,11 @@ export const getKnexDb = (config = {}) => {
|
|
|
52
56
|
}
|
|
53
57
|
return _knexDb;
|
|
54
58
|
};
|
|
55
|
-
export const isPostgres = () => {
|
|
56
|
-
return
|
|
57
|
-
};
|
|
58
|
-
export const isSqlite = () => {
|
|
59
|
-
return isDbType(SupportedDbType.sqlite);
|
|
59
|
+
export const isPostgres = (knex) => {
|
|
60
|
+
return knex.client.config.client === DbClient[SupportedDbType.postgres];
|
|
60
61
|
};
|
|
61
|
-
const
|
|
62
|
-
|
|
63
|
-
throw new Error("DB not initialized");
|
|
64
|
-
}
|
|
65
|
-
return _knexDbConfig.dbType === type;
|
|
62
|
+
export const isSqlite = (knex) => {
|
|
63
|
+
return knex.client.config.client === DbClient[SupportedDbType.sqlite];
|
|
66
64
|
};
|
|
67
65
|
const getMigrationConfig = () => {
|
|
68
66
|
return { directory: join(__dirname, "migrations") };
|
|
@@ -122,7 +120,9 @@ export const insertMultipleRows = (rows, table, columnNames, preventDuplicatesBy
|
|
|
122
120
|
// }
|
|
123
121
|
// }
|
|
124
122
|
const insertPlaceholders = generate2DPlaceholders(columnNames.length, rows.length);
|
|
125
|
-
const onConfict = preventDuplicatesByFirstColumn
|
|
123
|
+
const onConfict = preventDuplicatesByFirstColumn
|
|
124
|
+
? `ON CONFLICT (${columnNames[0]}) DO NOTHING`
|
|
125
|
+
: "";
|
|
126
126
|
yield getKnexDb().raw(`INSERT INTO ${table} (${columnNames.join(",")}) VALUES ${insertPlaceholders} ${onConfict}`, rows.flat());
|
|
127
127
|
return rows.length;
|
|
128
128
|
});
|
|
@@ -172,6 +172,13 @@ export function up(knex) {
|
|
|
172
172
|
.onDelete("CASCADE");
|
|
173
173
|
table_14.unique(["school_izo", "address_point_id"]);
|
|
174
174
|
});
|
|
175
|
+
yield knex.schema.createTable("sync_log", function (table_15) {
|
|
176
|
+
table_15.increments("id");
|
|
177
|
+
table_15.text("part").notNullable();
|
|
178
|
+
table_15.dateTime("started_at").notNullable();
|
|
179
|
+
table_15.dateTime("finished_at");
|
|
180
|
+
table_15.boolean("completed").defaultTo(false);
|
|
181
|
+
});
|
|
175
182
|
yield knex("founder_type").insert([
|
|
176
183
|
{ code: 0, name: "Zatím neurčeno" },
|
|
177
184
|
{ code: 101, name: "Fyzická osoba" },
|
|
@@ -218,16 +225,28 @@ export function up(knex) {
|
|
|
218
225
|
yield knex.schema.raw("CREATE INDEX municipality_part_code ON address_point (municipality_part_code);");
|
|
219
226
|
yield knex.schema.raw("CREATE INDEX object_type_id ON address_point (object_type_id);");
|
|
220
227
|
yield knex.schema.raw("CREATE INDEX street_code ON address_point (street_code);");
|
|
221
|
-
yield knex.schema.raw(`CREATE INDEX street_name ON street (name${isSqlite() ? " COLLATE NOCASE" : ""});`);
|
|
222
228
|
yield knex.schema.raw("CREATE INDEX street_sync_feed_url ON street_sync (feed_url);");
|
|
223
|
-
if (isPostgres()) {
|
|
229
|
+
if (isPostgres(knex)) {
|
|
224
230
|
yield knex.schema.raw("CREATE EXTENSION IF NOT EXISTS CITEXT");
|
|
225
|
-
yield knex.schema.alterTable("street", (t) => {
|
|
226
|
-
t.specificType("name", "CITEXT").notNullable().alter();
|
|
227
|
-
});
|
|
228
231
|
}
|
|
232
|
+
// case insensitive indexes
|
|
233
|
+
createCaseInsensitiveIndex(knex, "street", "name");
|
|
234
|
+
createCaseInsensitiveIndex(knex, "city", "name");
|
|
235
|
+
createCaseInsensitiveIndex(knex, "founder", "name");
|
|
236
|
+
createCaseInsensitiveIndex(knex, "founder", "short_name");
|
|
237
|
+
createCaseInsensitiveIndex(knex, "region", "name");
|
|
238
|
+
createCaseInsensitiveIndex(knex, "county", "name");
|
|
239
|
+
createCaseInsensitiveIndex(knex, "orp", "name");
|
|
229
240
|
});
|
|
230
241
|
}
|
|
242
|
+
const createCaseInsensitiveIndex = (knex, table, column) => __awaiter(void 0, void 0, void 0, function* () {
|
|
243
|
+
if (isPostgres(knex)) {
|
|
244
|
+
yield knex.schema.alterTable(table, (t) => {
|
|
245
|
+
t.specificType(column, "CITEXT").notNullable().alter();
|
|
246
|
+
});
|
|
247
|
+
}
|
|
248
|
+
yield knex.schema.raw(`CREATE INDEX ${table}_${column} ON ${table} (${column}${isSqlite(knex) ? " COLLATE NOCASE" : ""});`);
|
|
249
|
+
});
|
|
231
250
|
export function down(knex) {
|
|
232
251
|
return __awaiter(this, void 0, void 0, function* () {
|
|
233
252
|
yield knex.schema.dropTableIfExists("school_location");
|
|
@@ -247,5 +266,6 @@ export function down(knex) {
|
|
|
247
266
|
yield knex.schema.dropTableIfExists("prague_district");
|
|
248
267
|
yield knex.schema.dropTableIfExists("school");
|
|
249
268
|
yield knex.schema.dropTableIfExists("street_sync");
|
|
269
|
+
yield knex.schema.dropTableIfExists("sync_log");
|
|
250
270
|
});
|
|
251
271
|
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { SyncPart } from "./types";
|
|
2
|
+
export declare const startSyncPart: (part: SyncPart) => Promise<number>;
|
|
3
|
+
export declare const setSyncPartAsCompleted: (id: number) => Promise<void>;
|
|
4
|
+
export declare const isSyncPartCompleted: (part: SyncPart) => Promise<boolean>;
|
|
5
|
+
export declare const isEverythingSynced: () => Promise<boolean>;
|
|
6
|
+
export declare const millisecondsSinceLastSyncOfPart: (part: SyncPart) => Promise<number | null>;
|
|
7
|
+
export declare const lastSyncOfPart: (part: SyncPart) => Promise<Date | null>;
|
|
8
|
+
export declare const millisecondsSinceLastSync: () => Promise<number | null>;
|
|
9
|
+
export declare const lastSync: () => Promise<Date | null>;
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
2
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
3
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
4
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
5
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
6
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
7
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
8
|
+
});
|
|
9
|
+
};
|
|
10
|
+
import { getKnexDb, insertAutoincrementRow, isSqlite } from "./db";
|
|
11
|
+
import { SyncPart } from "./types";
|
|
12
|
+
const allParts = [
|
|
13
|
+
SyncPart.AddressPoints,
|
|
14
|
+
SyncPart.Schools,
|
|
15
|
+
SyncPart.Regions,
|
|
16
|
+
SyncPart.Streets,
|
|
17
|
+
];
|
|
18
|
+
const serializeDate = (date) => {
|
|
19
|
+
return isSqlite(getKnexDb()) ? date.toISOString().slice(0, -1) : date;
|
|
20
|
+
};
|
|
21
|
+
const deserializeDate = (date) => {
|
|
22
|
+
return isSqlite(getKnexDb()) ? new Date(date) : date;
|
|
23
|
+
};
|
|
24
|
+
export const startSyncPart = (part) => __awaiter(void 0, void 0, void 0, function* () {
|
|
25
|
+
return yield insertAutoincrementRow([part.toString(), serializeDate(new Date())], "sync_log", ["part", "started_at"]);
|
|
26
|
+
});
|
|
27
|
+
export const setSyncPartAsCompleted = (id) => __awaiter(void 0, void 0, void 0, function* () {
|
|
28
|
+
yield getKnexDb()
|
|
29
|
+
.update({ finished_at: serializeDate(new Date()), completed: true })
|
|
30
|
+
.from("sync_log")
|
|
31
|
+
.where({ id });
|
|
32
|
+
});
|
|
33
|
+
export const isSyncPartCompleted = (part) => __awaiter(void 0, void 0, void 0, function* () {
|
|
34
|
+
const result = yield getKnexDb()
|
|
35
|
+
.select("id")
|
|
36
|
+
.from("sync_log")
|
|
37
|
+
.where({ part, completed: true })
|
|
38
|
+
.limit(1);
|
|
39
|
+
return result.length > 0;
|
|
40
|
+
});
|
|
41
|
+
export const isEverythingSynced = () => __awaiter(void 0, void 0, void 0, function* () {
|
|
42
|
+
const results = yield Promise.all(allParts.map(isSyncPartCompleted));
|
|
43
|
+
return results.every((r) => r);
|
|
44
|
+
});
|
|
45
|
+
export const millisecondsSinceLastSyncOfPart = (part) => __awaiter(void 0, void 0, void 0, function* () {
|
|
46
|
+
const result = yield lastSyncOfPart(part);
|
|
47
|
+
return result === null ? null : new Date().getTime() - result.getTime();
|
|
48
|
+
});
|
|
49
|
+
export const lastSyncOfPart = (part) => __awaiter(void 0, void 0, void 0, function* () {
|
|
50
|
+
const result = yield getKnexDb()
|
|
51
|
+
.select("finished_at")
|
|
52
|
+
.from("sync_log")
|
|
53
|
+
.where({ part, completed: true })
|
|
54
|
+
.orderBy("finished_at", "desc")
|
|
55
|
+
.limit(1);
|
|
56
|
+
if (result.length === 0) {
|
|
57
|
+
return null;
|
|
58
|
+
}
|
|
59
|
+
return deserializeDate(result[0].finished_at);
|
|
60
|
+
});
|
|
61
|
+
export const millisecondsSinceLastSync = () => __awaiter(void 0, void 0, void 0, function* () {
|
|
62
|
+
const results = yield Promise.all(allParts.map(millisecondsSinceLastSyncOfPart));
|
|
63
|
+
if (results.some((r) => r === null)) {
|
|
64
|
+
return null;
|
|
65
|
+
}
|
|
66
|
+
return Math.max(...results);
|
|
67
|
+
});
|
|
68
|
+
export const lastSync = () => __awaiter(void 0, void 0, void 0, function* () {
|
|
69
|
+
const results = yield Promise.all(allParts.map(lastSyncOfPart));
|
|
70
|
+
if (results.some((r) => r === null)) {
|
|
71
|
+
return null;
|
|
72
|
+
}
|
|
73
|
+
return new Date(Math.min(...results.map((r) => r.getTime())));
|
|
74
|
+
});
|
package/dist/db/types.d.ts
CHANGED
package/dist/db/types.js
CHANGED
|
@@ -10,3 +10,10 @@ export const founderToMunicipality = (founder) => {
|
|
|
10
10
|
code: founder.cityOrDistrictCode,
|
|
11
11
|
};
|
|
12
12
|
};
|
|
13
|
+
export var SyncPart;
|
|
14
|
+
(function (SyncPart) {
|
|
15
|
+
SyncPart["AddressPoints"] = "address-points";
|
|
16
|
+
SyncPart["Schools"] = "schools";
|
|
17
|
+
SyncPart["Regions"] = "regions";
|
|
18
|
+
SyncPart["Streets"] = "streets";
|
|
19
|
+
})(SyncPart || (SyncPart = {}));
|
|
@@ -15,9 +15,10 @@ import fetch from "node-fetch";
|
|
|
15
15
|
import { join } from "path";
|
|
16
16
|
import { pipeline } from "stream/promises";
|
|
17
17
|
import { commitAddressPoints } from "../db/address-points";
|
|
18
|
-
import {
|
|
18
|
+
import { SyncPart } from "../db/types";
|
|
19
19
|
import { getLatestUrlFromAtomFeed } from "../utils/atom";
|
|
20
20
|
import { prepareOptions, } from "../utils/helpers";
|
|
21
|
+
import { runSyncPart } from "./common";
|
|
21
22
|
const maxBufferSize = 1000;
|
|
22
23
|
const downloadAndUnzip = (url, options) => __awaiter(void 0, void 0, void 0, function* () {
|
|
23
24
|
const zipFilePath = join(options.tmpDir, options.addressPointsZipFileName);
|
|
@@ -63,17 +64,11 @@ const importDataToDb = (options) => __awaiter(void 0, void 0, void 0, function*
|
|
|
63
64
|
});
|
|
64
65
|
const getExtractionFolder = (options) => join(options.tmpDir, options.addressPointsCsvFolderName);
|
|
65
66
|
export const downloadAndImportAddressPoints = (options = {}) => __awaiter(void 0, void 0, void 0, function* () {
|
|
66
|
-
|
|
67
|
+
yield runSyncPart(SyncPart.AddressPoints, [], () => __awaiter(void 0, void 0, void 0, function* () {
|
|
67
68
|
const completeOptions = prepareOptions(options);
|
|
68
69
|
const datasetFeedLink = yield getLatestUrlFromAtomFeed(completeOptions.addressPointsAtomUrl);
|
|
69
70
|
const zipUrl = yield getLatestUrlFromAtomFeed(datasetFeedLink);
|
|
70
|
-
|
|
71
|
+
yield downloadAndUnzip(zipUrl, completeOptions);
|
|
71
72
|
yield importDataToDb(completeOptions);
|
|
72
|
-
}
|
|
73
|
-
catch (error) {
|
|
74
|
-
console.error(error);
|
|
75
|
-
}
|
|
76
|
-
finally {
|
|
77
|
-
yield disconnectKnex();
|
|
78
|
-
}
|
|
73
|
+
}));
|
|
79
74
|
});
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
2
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
3
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
4
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
5
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
6
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
7
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
8
|
+
});
|
|
9
|
+
};
|
|
10
|
+
import { disconnectKnex } from "../db/db";
|
|
11
|
+
import { isSyncPartCompleted, setSyncPartAsCompleted, startSyncPart } from "../db/sync-log";
|
|
12
|
+
export const runSyncPart = (part, dependencies, partFunction) => __awaiter(void 0, void 0, void 0, function* () {
|
|
13
|
+
try {
|
|
14
|
+
const logId = yield startSyncPart(part);
|
|
15
|
+
for (const dependency of dependencies) {
|
|
16
|
+
const isCompleted = yield isSyncPartCompleted(dependency);
|
|
17
|
+
if (!isCompleted) {
|
|
18
|
+
throw new Error(`Sync part ${part} cannot be started because dependency ${dependency} is not completed. Run 'npm run ${dependency}' first.`);
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
yield partFunction();
|
|
22
|
+
yield setSyncPartAsCompleted(logId);
|
|
23
|
+
}
|
|
24
|
+
catch (error) {
|
|
25
|
+
console.log(error);
|
|
26
|
+
}
|
|
27
|
+
finally {
|
|
28
|
+
yield disconnectKnex();
|
|
29
|
+
}
|
|
30
|
+
});
|
|
@@ -12,8 +12,9 @@ import iconv from "iconv-lite";
|
|
|
12
12
|
import fetch from "node-fetch";
|
|
13
13
|
import { pipeline } from "stream/promises";
|
|
14
14
|
import { insertRegionsAndOrps } from "../db/regions";
|
|
15
|
+
import { SyncPart } from "../db/types";
|
|
15
16
|
import { prepareOptions } from "../utils/helpers";
|
|
16
|
-
import {
|
|
17
|
+
import { runSyncPart } from "./common";
|
|
17
18
|
const downloadAndImportDataToDb = (options) => __awaiter(void 0, void 0, void 0, function* () {
|
|
18
19
|
console.log("Downloading regions and ORP data...");
|
|
19
20
|
const response = yield fetch(options.regionsCsvUrl);
|
|
@@ -37,14 +38,8 @@ const downloadAndImportDataToDb = (options) => __awaiter(void 0, void 0, void 0,
|
|
|
37
38
|
console.log(`Import completed. Total imported rows: ${rows.length}`);
|
|
38
39
|
});
|
|
39
40
|
export const downloadAndImportRegions = (options = {}) => __awaiter(void 0, void 0, void 0, function* () {
|
|
40
|
-
|
|
41
|
+
yield runSyncPart(SyncPart.Regions, [], () => __awaiter(void 0, void 0, void 0, function* () {
|
|
41
42
|
const completeOptions = prepareOptions(options);
|
|
42
43
|
yield downloadAndImportDataToDb(completeOptions);
|
|
43
|
-
}
|
|
44
|
-
catch (error) {
|
|
45
|
-
console.log(error);
|
|
46
|
-
}
|
|
47
|
-
finally {
|
|
48
|
-
yield disconnectKnex();
|
|
49
|
-
}
|
|
44
|
+
}));
|
|
50
45
|
});
|
|
@@ -12,11 +12,11 @@ import fetch from "node-fetch";
|
|
|
12
12
|
import { join } from "path";
|
|
13
13
|
import sax from "sax";
|
|
14
14
|
import { pipeline } from "stream/promises";
|
|
15
|
-
import { disconnectKnex } from "../db/db";
|
|
16
15
|
import { insertFounders } from "../db/founders";
|
|
17
16
|
import { insertSchools } from "../db/schools";
|
|
18
|
-
import { MunicipalityType } from "../db/types";
|
|
17
|
+
import { MunicipalityType, SyncPart } from "../db/types";
|
|
19
18
|
import { prepareOptions, } from "../utils/helpers";
|
|
19
|
+
import { runSyncPart } from "./common";
|
|
20
20
|
const downloadXml = (options) => __awaiter(void 0, void 0, void 0, function* () {
|
|
21
21
|
if (existsSync(getXmlFilePath(options))) {
|
|
22
22
|
return;
|
|
@@ -300,18 +300,12 @@ const getXmlFilePath = (options) => {
|
|
|
300
300
|
return join(options.tmpDir, options.schoolsXmlFileName);
|
|
301
301
|
};
|
|
302
302
|
export const downloadAndImportSchools = (options, saveFoundersToCsv = false, saveSchoolsWithoutRuianToCsv = false) => __awaiter(void 0, void 0, void 0, function* () {
|
|
303
|
-
|
|
303
|
+
yield runSyncPart(SyncPart.Schools, [SyncPart.AddressPoints], () => __awaiter(void 0, void 0, void 0, function* () {
|
|
304
304
|
const runOptions = prepareOptions(options);
|
|
305
305
|
yield downloadXml(runOptions);
|
|
306
306
|
yield importDataToDb(runOptions, saveFoundersToCsv, saveSchoolsWithoutRuianToCsv);
|
|
307
307
|
deleteSchoolsXmlFile(runOptions);
|
|
308
|
-
}
|
|
309
|
-
catch (error) {
|
|
310
|
-
console.error(error);
|
|
311
|
-
}
|
|
312
|
-
finally {
|
|
313
|
-
yield disconnectKnex();
|
|
314
|
-
}
|
|
308
|
+
}));
|
|
315
309
|
});
|
|
316
310
|
export const deleteSchoolsXmlFile = (options = {}) => {
|
|
317
311
|
const runOptions = prepareOptions(options);
|
|
@@ -14,10 +14,10 @@ import parseDBF from "parsedbf";
|
|
|
14
14
|
import { join } from "path";
|
|
15
15
|
import { pipeline } from "stream/promises";
|
|
16
16
|
import { deleteStreets, getAllSyncedStreets, insertStreetsFromDbf, setStreetAsSynced, } from "../db/street-sync";
|
|
17
|
+
import { SyncPart } from "../db/types";
|
|
17
18
|
import { getAllUrlsFromAtomFeed, getLatestUrlFromAtomFeed, } from "../utils/atom";
|
|
18
19
|
import { prepareOptions, } from "../utils/helpers";
|
|
19
|
-
import {
|
|
20
|
-
import { areAddressPointsSynced } from "../db/address-points";
|
|
20
|
+
import { runSyncPart } from "./common";
|
|
21
21
|
const prepareFolders = (options) => {
|
|
22
22
|
const tempFolder = getTempFolder(options);
|
|
23
23
|
if (!existsSync(tempFolder)) {
|
|
@@ -48,11 +48,7 @@ const importDataToDb = (data) => __awaiter(void 0, void 0, void 0, function* ()
|
|
|
48
48
|
yield insertStreetsFromDbf(data);
|
|
49
49
|
});
|
|
50
50
|
export const downloadAndImportStreets = (options) => __awaiter(void 0, void 0, void 0, function* () {
|
|
51
|
-
|
|
52
|
-
// first check if address points are already synced
|
|
53
|
-
if (!(yield areAddressPointsSynced())) {
|
|
54
|
-
throw new Error("Address points are not synced yet. First run 'npm run address-points' and then try again.");
|
|
55
|
-
}
|
|
51
|
+
yield runSyncPart(SyncPart.Streets, [SyncPart.AddressPoints], () => __awaiter(void 0, void 0, void 0, function* () {
|
|
56
52
|
console.log("Starting to download and import streets. This takes up to 1 hour.");
|
|
57
53
|
const completeOptions = prepareOptions(options);
|
|
58
54
|
prepareFolders(completeOptions);
|
|
@@ -81,11 +77,5 @@ export const downloadAndImportStreets = (options) => __awaiter(void 0, void 0, v
|
|
|
81
77
|
done++;
|
|
82
78
|
console.log(`Loaded links: ${done}/${allDatasetFeedLinks.length}`);
|
|
83
79
|
}
|
|
84
|
-
}
|
|
85
|
-
catch (error) {
|
|
86
|
-
console.log(error);
|
|
87
|
-
}
|
|
88
|
-
finally {
|
|
89
|
-
yield disconnectKnex();
|
|
90
|
-
}
|
|
80
|
+
}));
|
|
91
81
|
});
|
package/package.json
CHANGED
|
@@ -39,6 +39,7 @@
|
|
|
39
39
|
"streets": "ts-node --experimental-specifier-resolution=node --esm src/bin/streets.ts",
|
|
40
40
|
"diagrams": "ts-node --experimental-specifier-resolution=node --esm src/bin/generate-syntax-diagrams.ts",
|
|
41
41
|
"clear-all": "ts-node --experimental-specifier-resolution=node --esm src/bin/clear-all.ts",
|
|
42
|
+
"init-db": "ts-node --experimental-specifier-resolution=node --esm src/bin/init-db.ts",
|
|
42
43
|
"parse-ordinance": "ts-node --experimental-specifier-resolution=node --esm src/bin/parse-ordinance.ts",
|
|
43
44
|
"knex": "ts-node --experimental-specifier-resolution=node --esm src/bin/knex.ts",
|
|
44
45
|
"test": "node --experimental-vm-modules --no-warnings node_modules/jest/bin/jest.js -c=jest-esm.config.mjs --no-cache",
|
|
@@ -52,7 +53,7 @@
|
|
|
52
53
|
},
|
|
53
54
|
"name": "text-to-map",
|
|
54
55
|
"description": "Text To Map usiluje o lepší, strojově zpracovatelné využití částí vyhlášek s výčtem ulic a dalších lokací. Jde o rozšiřitelnou sadu konceptů a nástrojů, které zajistí hladký převod výčtu ulic a jejich rozsahů v lidsky srozumitelném jazyce do strojově zpracovatelného, uchopitelného formátu.",
|
|
55
|
-
"version": "1.1.
|
|
56
|
+
"version": "1.1.4",
|
|
56
57
|
"main": "dist/index.js",
|
|
57
58
|
"types": "dist/index.d.ts",
|
|
58
59
|
"files": [
|