text-to-map 1.1.11 → 1.1.13

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (6) hide show
  1. package/dist/db/address-points.js +3 -2
  2. package/dist/db/db.d.ts +4 -1
  3. package/dist/db/db.js +33 -2
  4. package/dist/db/migrations/20230704125715_initial.js +87 -48
  5. package/dist/open-data-sync/address-points.js +11 -13
  6. package/package.json +4 -1
package/dist/db/address-points.js CHANGED
@@ -11,7 +11,7 @@ import { AddressPointType, createSingleLineAddress } from "czech-address";
11
11
  import { SeriesType, isNegativeSeriesSpec, isRange, isSeriesSpecArray, } from "../street-markdown/types";
12
12
  import { findClosestString } from "../utils/helpers";
13
13
  import jtsk2wgs84 from "../utils/jtsk2wgs84";
14
- import { extractKeyValuesPairs, generate2DPlaceholders, getKnexDb, insertMultipleRows, isSqlite, nonEmptyOrNull, } from "./db";
14
+ import { extractKeyValuesPairs, generate2DPlaceholders, getKnexDb, insertMultipleRows, isMysql, isSqlite, nonEmptyOrNull, } from "./db";
15
15
  import { getFounderCityCode } from "./founders";
16
16
  import { MunicipalityType } from "./types";
17
17
  const DescriptiveType = "č.p.";
@@ -79,7 +79,8 @@ export const commitAddressPoints = (buffer) => __awaiter(void 0, void 0, void 0,
79
79
  params.push(data[Column.admCode], nonEmptyOrNull(data[Column.streetCode]), ObjectTypes[data[Column.objectType]], nonEmptyOrNull(data[Column.houseNumber]), nonEmptyOrNull(data[Column.orientationalNumber]), nonEmptyOrNull(data[Column.orientationalNumberLetter]), data[Column.cityCode], nonEmptyOrNull(data[Column.districtCode]), nonEmptyOrNull(data[Column.municipalityPartCode]), nonEmptyOrNull(data[Column.pragueDistrictCode]), data[Column.postalCode], nonEmptyOrNull(data[Column.xCoordinate]), nonEmptyOrNull(data[Column.yCoordinate]), latOrNull, lonOrNull);
80
80
  });
81
81
  const placeHolders = generate2DPlaceholders(columnNames.length, buffer.length);
82
- yield getKnexDb().raw(`INSERT INTO address_point (${columnNames.join(",")}) VALUES ${placeHolders} ON CONFLICT (id) DO NOTHING`, params);
82
+ const knex = getKnexDb();
83
+ yield knex.raw(`INSERT ${isMysql(knex) ? "IGNORE" : ""} INTO address_point (${columnNames.join(",")}) VALUES ${placeHolders} ${!isMysql(knex) ? "ON CONFLICT (id) DO NOTHING" : ""}`, params);
83
84
  return buffer.length;
84
85
  });
85
86
  export const insertCities = (buffer) => __awaiter(void 0, void 0, void 0, function* () {
package/dist/db/db.d.ts CHANGED
@@ -2,17 +2,20 @@ import { Knex } from "knex";
2
2
  interface DbConfig {
3
3
  dbType: SupportedDbType;
4
4
  pgConnectionString?: string;
5
+ mysqlConnectionString?: string;
5
6
  initFilePath: string;
6
7
  filePath?: string;
7
8
  verbose: boolean;
8
9
  }
9
10
  export declare enum SupportedDbType {
10
11
  sqlite = "sqlite",
11
- postgres = "postgres"
12
+ postgres = "postgres",
13
+ mysql = "mysql"
12
14
  }
13
15
  export declare const getKnexDb: (config?: Partial<DbConfig>) => Knex;
14
16
  export declare const isPostgres: (knex: Knex) => boolean;
15
17
  export declare const isSqlite: (knex: Knex) => boolean;
18
+ export declare const isMysql: (knex: Knex) => boolean;
16
19
  export declare const initDb: (config?: Partial<DbConfig>) => Promise<Knex>;
17
20
  export declare const clearDb: (config?: Partial<DbConfig>) => Promise<Knex>;
18
21
  export declare const disconnectKnex: () => Promise<void>;
package/dist/db/db.js CHANGED
@@ -17,10 +17,13 @@ export var SupportedDbType;
17
17
  (function (SupportedDbType) {
18
18
  SupportedDbType["sqlite"] = "sqlite";
19
19
  SupportedDbType["postgres"] = "postgres";
20
+ SupportedDbType["mysql"] = "mysql";
20
21
  })(SupportedDbType || (SupportedDbType = {}));
22
+ const mysqlConnectionStringPattern = /^([^:]+):(\d+):([^:]+):([^:]+):([^:]+)$/;
21
23
  const DbClient = {
22
24
  [SupportedDbType.sqlite]: "better-sqlite3",
23
25
  [SupportedDbType.postgres]: "pg",
26
+ [SupportedDbType.mysql]: "mysql",
24
27
  };
25
28
  const defaults = {
26
29
  dbType: SupportedDbType.sqlite,
@@ -56,6 +59,20 @@ export const getKnexDb = (config = {}) => {
56
59
  useNullAsDefault: true,
57
60
  });
58
61
  }
62
+ else if (_knexDbConfig.dbType === SupportedDbType.mysql) {
63
+ const [host, port, user, password, database] = _knexDbConfig.mysqlConnectionString.split(":");
64
+ _knexDb = knex({
65
+ client: DbClient[SupportedDbType.mysql],
66
+ connection: {
67
+ host,
68
+ port: Number(port),
69
+ user,
70
+ password,
71
+ database,
72
+ },
73
+ useNullAsDefault: true,
74
+ });
75
+ }
59
76
  else {
60
77
  throw new Error(`Unsupported DB type: ${_knexDbConfig.dbType}`);
61
78
  }
@@ -68,6 +85,9 @@ export const isPostgres = (knex) => {
68
85
  export const isSqlite = (knex) => {
69
86
  return knex.client.config.client === DbClient[SupportedDbType.sqlite];
70
87
  };
88
+ export const isMysql = (knex) => {
89
+ return knex.client.config.client === DbClient[SupportedDbType.mysql];
90
+ };
71
91
  const getMigrationConfig = () => {
72
92
  return { directory: join(__dirname, "migrations") };
73
93
  };
@@ -100,6 +120,16 @@ const getEnvConfig = () => {
100
120
  }
101
121
  envConfig.pgConnectionString = process.env.TEXTTOMAP_PG_CONNECTION_STRING;
102
122
  }
123
+ else if (envConfig.dbType === SupportedDbType.mysql) {
124
+ if (!process.env.TEXTTOMAP_MYSQL_CONNECTION_DATA) {
125
+ throw new Error("Environmental variable 'TEXTTOMAP_MYSQL_CONNECTION_DATA' must be set for MySQL or MariaDB.");
126
+ }
127
+ if (!mysqlConnectionStringPattern.test(process.env.TEXTTOMAP_MYSQL_CONNECTION_DATA)) {
128
+ throw new Error("Environmental variable 'TEXTTOMAP_MYSQL_CONNECTION_DATA' must be in format 'host:port:user:password:database'.");
129
+ }
130
+ envConfig.mysqlConnectionString =
131
+ process.env.TEXTTOMAP_MYSQL_CONNECTION_DATA;
132
+ }
103
133
  else if (envConfig.dbType === SupportedDbType.sqlite) {
104
134
  if (process.env.TEXTTOMAP_SQLITE_PATH) {
105
135
  envConfig.filePath = process.env.TEXTTOMAP_SQLITE_PATH;
@@ -120,8 +150,9 @@ export const insertMultipleRows = (rows, table, columnNames, preventDuplicates =
120
150
  return 0;
121
151
  }
122
152
  const insertPlaceholders = generate2DPlaceholders(columnNames.length, rows.length);
123
- const onConfict = preventDuplicates ? `ON CONFLICT DO NOTHING` : "";
124
- yield getKnexDb().raw(`INSERT INTO ${table} (${columnNames.join(",")}) VALUES ${insertPlaceholders} ${onConfict}`, rows.flat());
153
+ const knex = getKnexDb();
154
+ const onConfict = preventDuplicates && !isMysql(knex) ? `ON CONFLICT DO NOTHING` : "";
155
+ yield knex.raw(`INSERT ${isMysql(knex) ? "IGNORE" : ""} INTO ${table} (${columnNames.join(",")}) VALUES ${insertPlaceholders} ${onConfict}`, rows.flat());
125
156
  return rows.length;
126
157
  });
127
158
  /**
package/dist/db/migrations/20230704125715_initial.js CHANGED
@@ -7,114 +7,130 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
7
7
  step((generator = generator.apply(thisArg, _arguments || [])).next());
8
8
  });
9
9
  };
10
- import { isPostgres, isSqlite } from "../db";
10
+ import { isMysql, isPostgres, isSqlite } from "../db";
11
11
  export function up(knex) {
12
12
  return __awaiter(this, void 0, void 0, function* () {
13
13
  yield knex.schema.createTable("prague_district", function (table_10) {
14
- table_10.integer("code").primary().notNullable();
15
- table_10.text("name").notNullable();
14
+ table_10.integer("code").unsigned().primary().notNullable();
15
+ table_10.string("name").notNullable();
16
16
  });
17
17
  yield knex.schema.createTable("school", function (table_12) {
18
- table_12.text("izo").primary();
19
- table_12.text("redizo").notNullable();
20
- table_12.text("name").notNullable();
21
- table_12.integer("capacity").notNullable();
18
+ table_12.string("izo").primary();
19
+ table_12.string("redizo").notNullable();
20
+ table_12.string("name").notNullable();
21
+ table_12.integer("capacity").unsigned().notNullable();
22
22
  });
23
23
  yield knex.schema.createTable("founder_type", function (table_5) {
24
- table_5.integer("code").primary().notNullable();
25
- table_5.text("name").notNullable();
24
+ table_5.integer("code").unsigned().primary().notNullable();
25
+ table_5.string("name").notNullable();
26
26
  });
27
27
  yield knex.schema.createTable("meta", function (table_6) {
28
- table_6.text("key").primary().notNullable();
28
+ table_6.string("key").primary().notNullable();
29
29
  table_6.text("value").notNullable();
30
30
  });
31
31
  yield knex.schema.createTable("object_type", function (table_8) {
32
- table_8.integer("id").primary().notNullable();
33
- table_8.text("name").notNullable();
32
+ table_8.integer("id").unsigned().primary().notNullable();
33
+ table_8.string("name").notNullable();
34
34
  });
35
35
  yield knex.schema.createTable("street_sync", function (table_16) {
36
- table_16.text("feed_url").primary().notNullable();
36
+ table_16.string("feed_url", 500).primary().notNullable();
37
37
  });
38
38
  yield knex.schema.createTable("region", function (table_11) {
39
- table_11.integer("code").primary().notNullable();
40
- table_11.text("name").notNullable();
41
- table_11.text("short_name").notNullable();
42
- table_11.integer("csu_code_100").notNullable();
43
- table_11.text("csu_code_108_nuts").notNullable();
39
+ table_11.integer("code").unsigned().primary().notNullable();
40
+ table_11.string("name").notNullable();
41
+ table_11.string("short_name").notNullable();
42
+ table_11.integer("csu_code_100").unsigned().notNullable();
43
+ table_11.string("csu_code_108_nuts").notNullable();
44
44
  });
45
45
  yield knex.schema.createTable("county", function (table_3) {
46
- table_3.integer("code").primary().notNullable();
47
- table_3.text("name").notNullable();
48
- table_3.integer("csu_code_101_lau").notNullable();
49
- table_3.text("csu_code_109_nuts").notNullable();
46
+ table_3.integer("code").unsigned().primary().notNullable();
47
+ table_3.string("name").notNullable();
48
+ table_3.integer("csu_code_101_lau").unsigned().notNullable();
49
+ table_3.string("csu_code_109_nuts").notNullable();
50
50
  table_3
51
51
  .integer("region_code")
52
+ .unsigned()
52
53
  .references("code")
53
54
  .inTable("region")
54
55
  .notNullable();
55
56
  });
56
57
  yield knex.schema.createTable("orp", function (table_9) {
57
- table_9.integer("code").primary().notNullable();
58
- table_9.text("name").notNullable();
59
- table_9.integer("csu_code_65").notNullable();
58
+ table_9.integer("code").unsigned().primary().notNullable();
59
+ table_9.string("name").notNullable();
60
+ table_9.integer("csu_code_65").unsigned().notNullable();
60
61
  table_9
61
62
  .integer("region_code")
63
+ .unsigned()
62
64
  .references("code")
63
65
  .inTable("region")
64
66
  .notNullable();
65
67
  table_9
66
68
  .integer("county_code")
69
+ .unsigned()
67
70
  .references("code")
68
71
  .inTable("county")
69
72
  .notNullable();
70
73
  });
71
74
  yield knex.schema.createTable("city", function (table_1) {
72
- table_1.integer("code").primary().notNullable();
73
- table_1.text("name").notNullable();
74
- table_1.integer("region_code").references("code").inTable("region");
75
- table_1.integer("county_code").references("code").inTable("county");
76
- table_1.integer("orp_code").references("code").inTable("orp");
75
+ table_1.integer("code").unsigned().primary().notNullable();
76
+ table_1.string("name").notNullable();
77
+ table_1
78
+ .integer("region_code")
79
+ .unsigned()
80
+ .references("code")
81
+ .inTable("region");
82
+ table_1
83
+ .integer("county_code")
84
+ .unsigned()
85
+ .references("code")
86
+ .inTable("county");
87
+ table_1.integer("orp_code").unsigned().references("code").inTable("orp");
77
88
  });
78
89
  yield knex.schema.createTable("city_district", function (table_2) {
79
- table_2.integer("code").primary().notNullable();
90
+ table_2.integer("code").unsigned().primary().notNullable();
80
91
  table_2
81
92
  .integer("city_code")
93
+ .unsigned()
82
94
  .references("code")
83
95
  .inTable("city")
84
96
  .notNullable();
85
- table_2.text("name").notNullable();
97
+ table_2.string("name").notNullable();
86
98
  });
87
99
  yield knex.schema.createTable("municipality_part", function (table_7) {
88
- table_7.integer("code").primary().notNullable();
100
+ table_7.integer("code").unsigned().primary().notNullable();
89
101
  table_7
90
102
  .integer("city_code")
103
+ .unsigned()
91
104
  .references("code")
92
105
  .inTable("city")
93
106
  .notNullable();
94
- table_7.text("name").notNullable();
107
+ table_7.string("name").notNullable();
95
108
  });
96
109
  yield knex.schema.createTable("street", function (table_15) {
97
- table_15.integer("code").primary().notNullable();
110
+ table_15.integer("code").unsigned().primary().notNullable();
98
111
  table_15
99
112
  .integer("city_code")
113
+ .unsigned()
100
114
  .references("code")
101
115
  .inTable("city")
102
116
  .notNullable();
103
- table_15.text("name").notNullable();
117
+ table_15.string("name").notNullable();
104
118
  });
105
119
  yield knex.schema.createTable("founder", function (table_4) {
106
120
  table_4.increments("id");
107
- table_4.text("name").notNullable();
108
- table_4.text("short_name").notNullable();
109
- table_4.text("ico").notNullable();
121
+ table_4.string("name").notNullable();
122
+ table_4.string("short_name").notNullable();
123
+ table_4.string("ico").notNullable();
110
124
  table_4
111
125
  .integer("founder_type_code")
126
+ .unsigned()
112
127
  .references("code")
113
128
  .inTable("founder_type")
114
129
  .notNullable();
115
- table_4.integer("city_code").references("code").inTable("city");
130
+ table_4.integer("city_code").unsigned().references("code").inTable("city");
116
131
  table_4
117
132
  .integer("city_district_code")
133
+ .unsigned()
118
134
  .references("code")
119
135
  .inTable("city_district");
120
136
  table_4.unique(["name", "ico"]);
@@ -122,12 +138,13 @@ export function up(knex) {
122
138
  yield knex.schema.createTable("school_founder", function (table_13) {
123
139
  table_13.increments("id");
124
140
  table_13
125
- .text("school_izo")
141
+ .string("school_izo")
126
142
  .references("izo")
127
143
  .inTable("school")
128
144
  .notNullable();
129
145
  table_13
130
146
  .integer("founder_id")
147
+ .unsigned()
131
148
  .references("id")
132
149
  .inTable("founder")
133
150
  .notNullable();
@@ -135,29 +152,42 @@ export function up(knex) {
135
152
  });
136
153
  yield knex.schema.createTable("address_point", function (table) {
137
154
  table.increments("id");
138
- table.integer("street_code").references("code").inTable("street");
155
+ table
156
+ .integer("street_code")
157
+ .unsigned()
158
+ .references("code")
159
+ .inTable("street");
139
160
  table
140
161
  .integer("object_type_id")
162
+ .unsigned()
141
163
  .references("id")
142
164
  .inTable("object_type")
143
165
  .notNullable();
144
166
  table.integer("descriptive_number");
145
167
  table.integer("orientational_number");
146
- table.text("orientational_number_letter");
147
- table.integer("city_code").references("code").inTable("city").notNullable();
168
+ table.string("orientational_number_letter");
169
+ table
170
+ .integer("city_code")
171
+ .unsigned()
172
+ .references("code")
173
+ .inTable("city")
174
+ .notNullable();
148
175
  table
149
176
  .integer("city_district_code")
177
+ .unsigned()
150
178
  .references("code")
151
179
  .inTable("city_district");
152
180
  table
153
181
  .integer("municipality_part_code")
182
+ .unsigned()
154
183
  .references("code")
155
184
  .inTable("municipality_part");
156
185
  table
157
186
  .integer("prague_district_code")
187
+ .unsigned()
158
188
  .references("code")
159
189
  .inTable("prague_district");
160
- table.text("postal_code").notNullable();
190
+ table.string("postal_code").notNullable();
161
191
  table.double("jtsk_x");
162
192
  table.double("jtsk_y");
163
193
  table.double("wgs84_latitude");
@@ -166,12 +196,13 @@ export function up(knex) {
166
196
  yield knex.schema.createTable("school_location", function (table_14) {
167
197
  table_14.increments("id");
168
198
  table_14
169
- .text("school_izo")
199
+ .string("school_izo")
170
200
  .references("izo")
171
201
  .inTable("school")
172
202
  .notNullable();
173
203
  table_14
174
204
  .integer("address_point_id")
205
+ .unsigned()
175
206
  .references("id")
176
207
  .inTable("address_point")
177
208
  .notNullable()
@@ -180,7 +211,7 @@ export function up(knex) {
180
211
  });
181
212
  yield knex.schema.createTable("sync_log", function (table_15) {
182
213
  table_15.increments("id");
183
- table_15.text("part").notNullable();
214
+ table_15.string("part").notNullable();
184
215
  table_15.dateTime("started_at").notNullable();
185
216
  table_15.dateTime("finished_at");
186
217
  table_15.boolean("completed").defaultTo(false);
@@ -246,12 +277,20 @@ export function up(knex) {
246
277
  });
247
278
  }
248
279
  const createCaseInsensitiveIndex = (knex, table, column) => __awaiter(void 0, void 0, void 0, function* () {
280
+ const createIndexStart = `CREATE INDEX ${table}_${column} ON ${table}`;
249
281
  if (isPostgres(knex)) {
250
282
  yield knex.schema.alterTable(table, (t) => {
251
283
  t.specificType(column, "CITEXT").notNullable().alter();
252
284
  });
285
+ yield knex.schema.raw(`${createIndexStart} (${column});`);
286
+ }
287
+ else if (isSqlite(knex)) {
288
+ yield knex.schema.raw(`${createIndexStart} (${column} COLLATE NOCASE);`);
289
+ }
290
+ else if (isMysql(knex)) {
291
+ yield knex.schema.raw(`ALTER TABLE ${table} MODIFY COLUMN ${column} VARCHAR(255) COLLATE utf8_general_ci;`);
292
+ yield knex.schema.raw(`${createIndexStart} (${column});`);
253
293
  }
254
- yield knex.schema.raw(`CREATE INDEX ${table}_${column} ON ${table} (${column}${isSqlite(knex) ? " COLLATE NOCASE" : ""});`);
255
294
  });
256
295
  export function down(knex) {
257
296
  return __awaiter(this, void 0, void 0, function* () {
package/dist/open-data-sync/address-points.js CHANGED
@@ -14,6 +14,7 @@ import iconv from "iconv-lite";
14
14
  import fetch from "node-fetch";
15
15
  import { join } from "path";
16
16
  import { pipeline } from "stream/promises";
17
+ import chunk from "lodash/chunk";
17
18
  import { commitAddressPoints } from "../db/address-points";
18
19
  import { SyncPart } from "../db/types";
19
20
  import { getLatestUrlFromAtomFeed } from "../utils/atom";
@@ -42,24 +43,21 @@ const importDataToDb = (options) => __awaiter(void 0, void 0, void 0, function*
42
43
  let total = 0;
43
44
  let next = 0;
44
45
  console.log("Initiating import of RUIAN data to search DB (~3 million rows to be imported).");
45
- const buffer = [];
46
46
  for (const file of files) {
47
+ const rows = [];
47
48
  const parseStream = parse({ delimiter: ";", fromLine: 2 }).on("data", (data) => __awaiter(void 0, void 0, void 0, function* () {
48
- buffer.push(data);
49
- if (buffer.length >= maxBufferSize) {
50
- parseStream.pause();
51
- total += yield commitAddressPoints(buffer);
52
- if (total - next >= 100000) {
53
- next += 100000;
54
- console.log(`Total imported rows: ${next}`);
55
- }
56
- buffer.length = 0;
57
- parseStream.resume();
58
- }
49
+ rows.push(data);
59
50
  }));
60
51
  yield pipeline(createReadStream(join(extractionFolder, file)), iconv.decodeStream("win1250"), parseStream);
52
+ const chunks = chunk(rows, maxBufferSize);
53
+ yield Promise.all(chunks.map((chunk) => __awaiter(void 0, void 0, void 0, function* () {
54
+ total += yield commitAddressPoints(chunk);
55
+ if (total - next >= 100000) {
56
+ next += 100000;
57
+ console.log(`Total imported rows: ${next}`);
58
+ }
59
+ })));
61
60
  }
62
- total += yield commitAddressPoints(buffer);
63
61
  console.log(`Import completed. Total imported rows: ${total}`);
64
62
  });
65
63
  const getExtractionFolder = (options) => join(options.tmpDir, options.addressPointsCsvFolderName);
package/package.json CHANGED
@@ -11,6 +11,8 @@
11
11
  "feedparser": "^2.2.10",
12
12
  "iconv-lite": "^0.6.3",
13
13
  "knex": "^2.4.2",
14
+ "lodash": "^4.17.21",
15
+ "mysql": "^2.18.1",
14
16
  "node-fetch": "^3.3.0",
15
17
  "parsedbf": "^1.1.1",
16
18
  "pg": "^8.11.1",
@@ -23,6 +25,7 @@
23
25
  "@types/adm-zip": "^0.5.0",
24
26
  "@types/better-sqlite3": "^7.6.3",
25
27
  "@types/feedparser": "^2.2.5",
28
+ "@types/lodash": "^4.14.196",
26
29
  "@types/node": "^18.11.17",
27
30
  "copyfiles": "^2.4.1",
28
31
  "jest": "^29.5.0",
@@ -53,7 +56,7 @@
53
56
  },
54
57
  "name": "text-to-map",
55
58
  "description": "Text To Map usiluje o lepší, strojově zpracovatelné využití částí vyhlášek s výčtem ulic a dalších lokací. Jde o rozšiřitelnou sadu konceptů a nástrojů, které zajistí hladký převod výčtu ulic a jejich rozsahů v lidsky srozumitelném jazyce do strojově zpracovatelného, uchopitelného formátu.",
56
- "version": "1.1.11",
59
+ "version": "1.1.13",
57
60
  "main": "dist/index.js",
58
61
  "types": "dist/index.d.ts",
59
62
  "files": [