text-to-map 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +20 -0
- package/dist/bin/address-points-import-only.d.ts +1 -0
- package/dist/bin/address-points-import-only.js +14 -0
- package/dist/bin/address-points.d.ts +1 -0
- package/dist/bin/address-points.js +14 -0
- package/dist/bin/clear-all.d.ts +1 -0
- package/dist/bin/clear-all.js +5 -0
- package/dist/bin/generate-syntax-diagrams.d.ts +1 -0
- package/dist/bin/generate-syntax-diagrams.js +19 -0
- package/dist/bin/parse-ordinance.d.ts +1 -0
- package/dist/bin/parse-ordinance.js +20 -0
- package/dist/bin/schools.d.ts +1 -0
- package/dist/bin/schools.js +8 -0
- package/dist/bin/streets.d.ts +1 -0
- package/dist/bin/streets.js +14 -0
- package/dist/db/address-points.d.ts +38 -0
- package/dist/db/address-points.js +312 -0
- package/dist/db/db.d.ts +23 -0
- package/dist/db/db.js +90 -0
- package/dist/db/founders.d.ts +9 -0
- package/dist/db/founders.js +191 -0
- package/dist/db/meta.d.ts +3 -0
- package/dist/db/meta.js +17 -0
- package/dist/db/schools.d.ts +6 -0
- package/dist/db/schools.js +61 -0
- package/dist/db/street-sync.d.ts +5 -0
- package/dist/db/street-sync.js +18 -0
- package/dist/db/types.d.ts +34 -0
- package/dist/db/types.js +12 -0
- package/dist/index.d.ts +4 -0
- package/dist/index.js +4 -0
- package/dist/open-data-sync/address-points.d.ts +4 -0
- package/dist/open-data-sync/address-points.js +73 -0
- package/dist/open-data-sync/schools.d.ts +3 -0
- package/dist/open-data-sync/schools.js +308 -0
- package/dist/open-data-sync/streets.d.ts +2 -0
- package/dist/open-data-sync/streets.js +77 -0
- package/dist/smd-to-csv/json-to-csv.d.ts +1 -0
- package/dist/smd-to-csv/json-to-csv.js +153 -0
- package/dist/smd-to-csv/ordinance-to-csv-rules.d.ts +1 -0
- package/dist/smd-to-csv/ordinance-to-csv-rules.js +31 -0
- package/dist/smd-to-csv/txt-to-json.d.ts +1 -0
- package/dist/smd-to-csv/txt-to-json.js +102 -0
- package/dist/street-markdown/municipality.d.ts +5 -0
- package/dist/street-markdown/municipality.js +44 -0
- package/dist/street-markdown/roman-numerals.d.ts +1 -0
- package/dist/street-markdown/roman-numerals.js +67 -0
- package/dist/street-markdown/smd-line-parser.d.ts +4 -0
- package/dist/street-markdown/smd-line-parser.js +29 -0
- package/dist/street-markdown/smd-parser.d.ts +21 -0
- package/dist/street-markdown/smd-parser.js +343 -0
- package/dist/street-markdown/smd.d.ts +9 -0
- package/dist/street-markdown/smd.js +182 -0
- package/dist/street-markdown/token-definition.d.ts +23 -0
- package/dist/street-markdown/token-definition.js +154 -0
- package/dist/street-markdown/types.d.ts +83 -0
- package/dist/street-markdown/types.js +18 -0
- package/dist/utils/addressBuilder.d.ts +8 -0
- package/dist/utils/addressBuilder.js +101 -0
- package/dist/utils/atom.d.ts +2 -0
- package/dist/utils/atom.js +65 -0
- package/dist/utils/helpers.d.ts +34 -0
- package/dist/utils/helpers.js +157 -0
- package/package.json +82 -0
|
@@ -0,0 +1,308 @@
|
|
|
1
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
2
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
3
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
4
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
5
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
6
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
7
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
8
|
+
});
|
|
9
|
+
};
|
|
10
|
+
import { createReadStream, createWriteStream, existsSync, rmSync } from "fs";
|
|
11
|
+
import { join } from "path";
|
|
12
|
+
import sax from "sax";
|
|
13
|
+
import fetch from "node-fetch";
|
|
14
|
+
import { setDbConfig } from "../db/db";
|
|
15
|
+
import { prepareOptions, } from "../utils/helpers";
|
|
16
|
+
import { pipeline } from "stream/promises";
|
|
17
|
+
import { MunicipalityType } from "../db/types";
|
|
18
|
+
import { insertSchools } from "../db/schools";
|
|
19
|
+
import { insertFounders } from "../db/founders";
|
|
20
|
+
const downloadXml = (options) => __awaiter(void 0, void 0, void 0, function* () {
|
|
21
|
+
if (existsSync(getXmlFilePath(options))) {
|
|
22
|
+
return;
|
|
23
|
+
}
|
|
24
|
+
console.log("Downloading a large XML file with school data...");
|
|
25
|
+
const response = yield fetch(options.schoolsXmlUrl);
|
|
26
|
+
if (response.status !== 200) {
|
|
27
|
+
throw new Error(`The XML file could not be downloaded. HTTP Code: ${response.status}`);
|
|
28
|
+
}
|
|
29
|
+
yield pipeline(response.body, createWriteStream(getXmlFilePath(options)));
|
|
30
|
+
console.log("Finished downloading.");
|
|
31
|
+
});
|
|
32
|
+
var XMLState;
|
|
33
|
+
(function (XMLState) {
|
|
34
|
+
XMLState[XMLState["None"] = 0] = "None";
|
|
35
|
+
XMLState[XMLState["SchoolName"] = 1] = "SchoolName";
|
|
36
|
+
XMLState[XMLState["Izo"] = 2] = "Izo";
|
|
37
|
+
XMLState[XMLState["Ico"] = 3] = "Ico";
|
|
38
|
+
XMLState[XMLState["SchoolType"] = 4] = "SchoolType";
|
|
39
|
+
XMLState[XMLState["Capacity"] = 5] = "Capacity";
|
|
40
|
+
XMLState[XMLState["RuianCode"] = 6] = "RuianCode";
|
|
41
|
+
XMLState[XMLState["Address"] = 7] = "Address";
|
|
42
|
+
XMLState[XMLState["FounderName"] = 8] = "FounderName";
|
|
43
|
+
XMLState[XMLState["FounderType"] = 9] = "FounderType";
|
|
44
|
+
XMLState[XMLState["FounderIco"] = 10] = "FounderIco";
|
|
45
|
+
})(XMLState || (XMLState = {}));
|
|
46
|
+
const SCHOOL_TYPE_PRIMARY = "B00";
|
|
47
|
+
const createNewSchool = () => {
|
|
48
|
+
return {
|
|
49
|
+
name: "",
|
|
50
|
+
izo: "",
|
|
51
|
+
capacity: 0,
|
|
52
|
+
locations: [],
|
|
53
|
+
};
|
|
54
|
+
};
|
|
55
|
+
const getCorrectFounderType = (founderType) => {
|
|
56
|
+
return founderType === "" ? "101" : founderType;
|
|
57
|
+
};
|
|
58
|
+
const getMunicipalityType = (founderType) => {
|
|
59
|
+
return founderType === "261"
|
|
60
|
+
? MunicipalityType.City
|
|
61
|
+
: founderType === "263"
|
|
62
|
+
? MunicipalityType.District
|
|
63
|
+
: MunicipalityType.Other;
|
|
64
|
+
};
|
|
65
|
+
const processSchoolRegisterXml = (options) => __awaiter(void 0, void 0, void 0, function* () {
|
|
66
|
+
let currentSchool;
|
|
67
|
+
let isCurrentSchoolPrimary;
|
|
68
|
+
let currentIzo;
|
|
69
|
+
let currentIco;
|
|
70
|
+
let currentType;
|
|
71
|
+
let currentCapacity;
|
|
72
|
+
let currentLocations = [];
|
|
73
|
+
let state = XMLState.None;
|
|
74
|
+
let currentFounders = [];
|
|
75
|
+
let currentFounderIco;
|
|
76
|
+
let currentFounderName;
|
|
77
|
+
let currentFounderType;
|
|
78
|
+
let isRuianCodeSet = false;
|
|
79
|
+
let isRuianCodeMissing = false;
|
|
80
|
+
let currentAddress = [];
|
|
81
|
+
const founders = new Map();
|
|
82
|
+
const schools = [];
|
|
83
|
+
const schoolsWithoutRuian = [];
|
|
84
|
+
const streamPromise = new Promise((resolve, reject) => {
|
|
85
|
+
const saxStream = sax
|
|
86
|
+
.createStream(true)
|
|
87
|
+
.on("opentag", (tag) => {
|
|
88
|
+
switch (tag.name) {
|
|
89
|
+
case "PravniSubjekt":
|
|
90
|
+
currentSchool = createNewSchool();
|
|
91
|
+
isCurrentSchoolPrimary = false;
|
|
92
|
+
isRuianCodeMissing = false;
|
|
93
|
+
break;
|
|
94
|
+
case "RedPlnyNazev":
|
|
95
|
+
state = XMLState.SchoolName;
|
|
96
|
+
break;
|
|
97
|
+
case "IZO":
|
|
98
|
+
state = XMLState.Izo;
|
|
99
|
+
break;
|
|
100
|
+
case "ICO":
|
|
101
|
+
state = XMLState.Ico;
|
|
102
|
+
break;
|
|
103
|
+
case "SkolaDruhTyp":
|
|
104
|
+
state = XMLState.SchoolType;
|
|
105
|
+
break;
|
|
106
|
+
case "SkolaKapacita":
|
|
107
|
+
state = XMLState.Capacity;
|
|
108
|
+
break;
|
|
109
|
+
case "MistoRUAINKod":
|
|
110
|
+
isRuianCodeSet = false;
|
|
111
|
+
currentAddress = [];
|
|
112
|
+
state = XMLState.RuianCode;
|
|
113
|
+
break;
|
|
114
|
+
case "ZrizNazev":
|
|
115
|
+
state = XMLState.FounderName;
|
|
116
|
+
break;
|
|
117
|
+
case "ZrizPravniForma":
|
|
118
|
+
state = XMLState.FounderType;
|
|
119
|
+
break;
|
|
120
|
+
case "MistoAdresa1":
|
|
121
|
+
case "MistoAdresa2":
|
|
122
|
+
case "MistoAdresa3":
|
|
123
|
+
state = XMLState.Address;
|
|
124
|
+
break;
|
|
125
|
+
case "ZrizDatumNarozeni":
|
|
126
|
+
case "ZrizICO":
|
|
127
|
+
state = XMLState.FounderIco;
|
|
128
|
+
break;
|
|
129
|
+
}
|
|
130
|
+
})
|
|
131
|
+
.on("closetag", (tagName) => {
|
|
132
|
+
switch (tagName) {
|
|
133
|
+
case "PravniSubjekt":
|
|
134
|
+
if (isCurrentSchoolPrimary) {
|
|
135
|
+
schools.push(currentSchool);
|
|
136
|
+
currentFounders.forEach((founder) => {
|
|
137
|
+
const key = founder.name + founder.ico;
|
|
138
|
+
if (founders.has(key)) {
|
|
139
|
+
founders.get(key).schools.push(currentSchool);
|
|
140
|
+
}
|
|
141
|
+
else {
|
|
142
|
+
founders.set(key, {
|
|
143
|
+
name: founder.name,
|
|
144
|
+
ico: founder.ico,
|
|
145
|
+
originalType: founder.type,
|
|
146
|
+
municipalityType: getMunicipalityType(founder.type),
|
|
147
|
+
schools: [currentSchool],
|
|
148
|
+
});
|
|
149
|
+
}
|
|
150
|
+
});
|
|
151
|
+
}
|
|
152
|
+
currentIco = "";
|
|
153
|
+
currentFounders = [];
|
|
154
|
+
break;
|
|
155
|
+
case "Zrizovatel":
|
|
156
|
+
if (currentFounderIco === "" || currentFounderName === "") {
|
|
157
|
+
currentFounders.push({
|
|
158
|
+
ico: currentIco,
|
|
159
|
+
name: currentSchool.name,
|
|
160
|
+
type: "224", // s.r.o (not all are those, but we don't need to differentiate here)
|
|
161
|
+
});
|
|
162
|
+
}
|
|
163
|
+
else {
|
|
164
|
+
currentFounders.push({
|
|
165
|
+
ico: currentFounderIco,
|
|
166
|
+
name: currentFounderName,
|
|
167
|
+
type: getCorrectFounderType(currentFounderType),
|
|
168
|
+
});
|
|
169
|
+
}
|
|
170
|
+
currentFounderIco = "";
|
|
171
|
+
currentFounderName = "";
|
|
172
|
+
currentFounderType = "";
|
|
173
|
+
break;
|
|
174
|
+
case "SkolaZarizeni":
|
|
175
|
+
if (currentType === SCHOOL_TYPE_PRIMARY) {
|
|
176
|
+
currentSchool.izo = currentIzo;
|
|
177
|
+
currentSchool.locations = currentLocations;
|
|
178
|
+
currentSchool.capacity = currentCapacity;
|
|
179
|
+
}
|
|
180
|
+
currentLocations = [];
|
|
181
|
+
break;
|
|
182
|
+
case "SkolaMistoVykonuCinnosti":
|
|
183
|
+
if (isRuianCodeMissing) {
|
|
184
|
+
schoolsWithoutRuian.push({
|
|
185
|
+
izo: currentIzo,
|
|
186
|
+
address: currentAddress,
|
|
187
|
+
isPrimary: currentType === SCHOOL_TYPE_PRIMARY,
|
|
188
|
+
});
|
|
189
|
+
}
|
|
190
|
+
break;
|
|
191
|
+
case "MistoRUAINKod":
|
|
192
|
+
isRuianCodeMissing = !isRuianCodeSet;
|
|
193
|
+
case "RedPlnyNazev":
|
|
194
|
+
case "ICO":
|
|
195
|
+
case "IZO":
|
|
196
|
+
case "SkolaDruhTyp":
|
|
197
|
+
case "SkolaKapacita":
|
|
198
|
+
case "ZrizNazev":
|
|
199
|
+
case "ZrizICO":
|
|
200
|
+
case "ZrizDatumNarozeni":
|
|
201
|
+
case "ZrizPravniForma":
|
|
202
|
+
case "MistoAdresa1":
|
|
203
|
+
case "MistoAdresa2":
|
|
204
|
+
case "MistoAdresa3":
|
|
205
|
+
state = XMLState.None;
|
|
206
|
+
break;
|
|
207
|
+
}
|
|
208
|
+
})
|
|
209
|
+
.on("text", (text) => {
|
|
210
|
+
switch (state) {
|
|
211
|
+
case XMLState.SchoolName:
|
|
212
|
+
currentSchool.name = text;
|
|
213
|
+
break;
|
|
214
|
+
case XMLState.Izo:
|
|
215
|
+
currentIzo = text;
|
|
216
|
+
break;
|
|
217
|
+
case XMLState.Ico:
|
|
218
|
+
currentIco = text;
|
|
219
|
+
break;
|
|
220
|
+
case XMLState.SchoolType:
|
|
221
|
+
currentType = text;
|
|
222
|
+
if (text === SCHOOL_TYPE_PRIMARY) {
|
|
223
|
+
isCurrentSchoolPrimary = true;
|
|
224
|
+
}
|
|
225
|
+
break;
|
|
226
|
+
case XMLState.RuianCode:
|
|
227
|
+
isRuianCodeSet = true;
|
|
228
|
+
currentLocations.push({
|
|
229
|
+
addressPointId: parseInt(text),
|
|
230
|
+
});
|
|
231
|
+
break;
|
|
232
|
+
case XMLState.Address:
|
|
233
|
+
currentAddress.push(text);
|
|
234
|
+
break;
|
|
235
|
+
case XMLState.FounderName:
|
|
236
|
+
currentFounderName = text;
|
|
237
|
+
break;
|
|
238
|
+
case XMLState.FounderIco:
|
|
239
|
+
currentFounderIco = text;
|
|
240
|
+
break;
|
|
241
|
+
case XMLState.FounderType:
|
|
242
|
+
currentFounderType = text;
|
|
243
|
+
break;
|
|
244
|
+
case XMLState.Capacity:
|
|
245
|
+
currentCapacity = parseInt(text);
|
|
246
|
+
break;
|
|
247
|
+
}
|
|
248
|
+
})
|
|
249
|
+
.on("error", reject)
|
|
250
|
+
.on("end", resolve);
|
|
251
|
+
// wanted to use 'await pipeline(createReadStream(getXmlFilePath(options)), saxStream)'
|
|
252
|
+
// but the program would quit spontaneously after finishing stream - using Promise instead.
|
|
253
|
+
createReadStream(getXmlFilePath(options)).pipe(saxStream);
|
|
254
|
+
});
|
|
255
|
+
yield streamPromise;
|
|
256
|
+
return { schools, founders, schoolsWithoutRuian };
|
|
257
|
+
});
|
|
258
|
+
const importDataToDb = (options, saveFoundersToCsv = false, saveSchoolsWithoutRuianToCsv = false) => __awaiter(void 0, void 0, void 0, function* () {
|
|
259
|
+
const { schools, founders, schoolsWithoutRuian } = yield processSchoolRegisterXml(options);
|
|
260
|
+
if (saveFoundersToCsv) {
|
|
261
|
+
const csvFile = "founders.csv";
|
|
262
|
+
if (existsSync(csvFile)) {
|
|
263
|
+
rmSync(csvFile);
|
|
264
|
+
}
|
|
265
|
+
var csv = createWriteStream(csvFile, {
|
|
266
|
+
flags: "a",
|
|
267
|
+
});
|
|
268
|
+
csv.write("IČO;Zřizovatel;Právní forma;Počet škol;Školy\n");
|
|
269
|
+
founders.forEach((founder) => {
|
|
270
|
+
csv.write(`#${founder.ico};${founder.name};${founder.originalType};${founder.schools.length};${founder.schools.map((school) => school.name).join("---")}\n`);
|
|
271
|
+
});
|
|
272
|
+
csv.end();
|
|
273
|
+
}
|
|
274
|
+
if (saveSchoolsWithoutRuianToCsv) {
|
|
275
|
+
const csvFile = "schoolsWithoutRuian.csv";
|
|
276
|
+
if (existsSync(csvFile)) {
|
|
277
|
+
rmSync(csvFile);
|
|
278
|
+
}
|
|
279
|
+
var csv = createWriteStream(csvFile, {
|
|
280
|
+
flags: "a",
|
|
281
|
+
});
|
|
282
|
+
csv.write("IZO;Je základní;adresa1;adresa2;adresa3\n");
|
|
283
|
+
schoolsWithoutRuian.forEach((schoolAddress) => {
|
|
284
|
+
csv.write(`#${schoolAddress.izo};${schoolAddress.isPrimary ? "TRUE" : "FALSE"};${schoolAddress.address.join(";")}\n`);
|
|
285
|
+
});
|
|
286
|
+
csv.end();
|
|
287
|
+
}
|
|
288
|
+
setDbConfig({
|
|
289
|
+
filePath: options.dbFilePath,
|
|
290
|
+
initFilePath: options.dbInitFilePath,
|
|
291
|
+
});
|
|
292
|
+
insertSchools(schools);
|
|
293
|
+
insertFounders(Array.from(founders.values()));
|
|
294
|
+
});
|
|
295
|
+
const getXmlFilePath = (options) => {
|
|
296
|
+
return join(options.tmpDir, options.schoolsXmlFileName);
|
|
297
|
+
};
|
|
298
|
+
export const downloadAndImportAllSchools = (options) => __awaiter(void 0, void 0, void 0, function* () {
|
|
299
|
+
const runOptions = prepareOptions(options);
|
|
300
|
+
yield downloadXml(runOptions);
|
|
301
|
+
yield importDataToDb(runOptions, false, true);
|
|
302
|
+
});
|
|
303
|
+
export const deleteSchoolsXmlFile = (options = {}) => {
|
|
304
|
+
const runOptions = prepareOptions(options);
|
|
305
|
+
if (existsSync(getXmlFilePath(runOptions))) {
|
|
306
|
+
rmSync(getXmlFilePath(runOptions));
|
|
307
|
+
}
|
|
308
|
+
};
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
2
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
3
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
4
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
5
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
6
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
7
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
8
|
+
});
|
|
9
|
+
};
|
|
10
|
+
import fetch from "node-fetch";
|
|
11
|
+
import { createWriteStream, rmSync, existsSync, mkdirSync } from "fs";
|
|
12
|
+
import { pipeline } from "stream/promises";
|
|
13
|
+
import { join } from "path";
|
|
14
|
+
import AdmZip from "adm-zip";
|
|
15
|
+
import parseDBF from "parsedbf";
|
|
16
|
+
import { prepareOptions, initDb, } from "../utils/helpers";
|
|
17
|
+
import { deleteStreets, getAllSyncedStreets, insertStreetsFromDbf, setStreetAsSynced, } from "../db/street-sync";
|
|
18
|
+
import { getAllUrlsFromAtomFeed, getLatestUrlFromAtomFeed, } from "../utils/atom";
|
|
19
|
+
const prepareFolders = (options) => {
|
|
20
|
+
const tempFolder = getTempFolder(options);
|
|
21
|
+
if (!existsSync(tempFolder)) {
|
|
22
|
+
mkdirSync(tempFolder);
|
|
23
|
+
}
|
|
24
|
+
};
|
|
25
|
+
const getTempFolder = (options) => {
|
|
26
|
+
return join(options.tmpDir, options.streetFolderName);
|
|
27
|
+
};
|
|
28
|
+
const downloadZipAndParseDbfFile = (url, index, options) => __awaiter(void 0, void 0, void 0, function* () {
|
|
29
|
+
const response = yield fetch(url);
|
|
30
|
+
if (response.status !== 200) {
|
|
31
|
+
throw new Error(`The ZIP file from '${url}' could not be downloaded. HTTP Code: ${response.status}`);
|
|
32
|
+
}
|
|
33
|
+
const zipFilePath = join(getTempFolder(options), `street${index}.zip`);
|
|
34
|
+
yield pipeline(response.body, createWriteStream(zipFilePath));
|
|
35
|
+
const zip = new AdmZip(zipFilePath);
|
|
36
|
+
const folderName = zip.getEntries()[0].entryName;
|
|
37
|
+
const dbfEntryName = `${folderName}${options.streetDbfFileName}`;
|
|
38
|
+
const obj = parseDBF(zip.getEntry(dbfEntryName).getData(), "win1250");
|
|
39
|
+
rmSync(zipFilePath);
|
|
40
|
+
return obj;
|
|
41
|
+
});
|
|
42
|
+
const importDataToDb = (data, options) => __awaiter(void 0, void 0, void 0, function* () {
|
|
43
|
+
if (data.length === 0) {
|
|
44
|
+
return;
|
|
45
|
+
}
|
|
46
|
+
insertStreetsFromDbf(data);
|
|
47
|
+
});
|
|
48
|
+
export const downloadAndImportAllStreets = (options) => __awaiter(void 0, void 0, void 0, function* () {
|
|
49
|
+
const completeOptions = prepareOptions(options);
|
|
50
|
+
initDb(completeOptions);
|
|
51
|
+
prepareFolders(completeOptions);
|
|
52
|
+
const allDatasetFeedLinks = yield getAllUrlsFromAtomFeed(completeOptions.streetsAtomUrl);
|
|
53
|
+
console.log(`Total of ${allDatasetFeedLinks.length} links to ZIP files.`);
|
|
54
|
+
const syncedStreetLinks = getAllSyncedStreets();
|
|
55
|
+
const toDelete = [];
|
|
56
|
+
// remove all deprecated links (not in the new list)
|
|
57
|
+
syncedStreetLinks.forEach((link) => {
|
|
58
|
+
if (!allDatasetFeedLinks.includes(link)) {
|
|
59
|
+
toDelete.push(link);
|
|
60
|
+
delete syncedStreetLinks[link];
|
|
61
|
+
}
|
|
62
|
+
});
|
|
63
|
+
deleteStreets(toDelete);
|
|
64
|
+
let done = syncedStreetLinks.size;
|
|
65
|
+
console.log(`Total of ${done} links to ZIP files already stored.`);
|
|
66
|
+
// get all links not yet stored
|
|
67
|
+
const newLinks = allDatasetFeedLinks.filter((link) => !syncedStreetLinks.has(link));
|
|
68
|
+
console.log(`Loading ${newLinks.length} new links to ZIP files.`);
|
|
69
|
+
for (const link of newLinks) {
|
|
70
|
+
const zipLink = yield getLatestUrlFromAtomFeed(link);
|
|
71
|
+
const dbfObject = yield downloadZipAndParseDbfFile(zipLink, done, completeOptions);
|
|
72
|
+
yield importDataToDb(dbfObject, completeOptions);
|
|
73
|
+
setStreetAsSynced(link);
|
|
74
|
+
done++;
|
|
75
|
+
console.log(`Loaded links: ${done}/${allDatasetFeedLinks.length}`);
|
|
76
|
+
}
|
|
77
|
+
});
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export function districtsToCsvRules(districts: any, includeDistrictName?: boolean): string;
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
import { readFileSync } from "fs";
|
|
2
|
+
const streetNamePattern = /^([^ -]+([ -]?[^ -]+)*)/;
|
|
3
|
+
const streetSeriesPartPattern = /(lichá č.|sudá č.|č. p.|č.)/;
|
|
4
|
+
const rangePattern = /(\d+[a-z]? ?[-] ?\d+[a-z]?|(od )?\d+[a-z]?( a)? výše|\d+[a-z]?)(, ?| a )/;
|
|
5
|
+
const numberPattern = /\d+/;
|
|
6
|
+
const OddTypeString = "lichá č.";
|
|
7
|
+
const EvenTypeString = "sudá č.";
|
|
8
|
+
const AllTypeString = "č.";
|
|
9
|
+
const CPTypeString = "č. p.";
|
|
10
|
+
const StreetNumberSeriesType = {
|
|
11
|
+
Odd: "odd",
|
|
12
|
+
Even: "even",
|
|
13
|
+
CP: "cp",
|
|
14
|
+
All: "all",
|
|
15
|
+
};
|
|
16
|
+
const _min = 0;
|
|
17
|
+
const _max = 10000;
|
|
18
|
+
const StreetNumberRule = {
|
|
19
|
+
MinFrom: _min,
|
|
20
|
+
MaxTo: _max,
|
|
21
|
+
EmptyRule: {
|
|
22
|
+
type: StreetNumberSeriesType.All,
|
|
23
|
+
from: _min,
|
|
24
|
+
to: _max,
|
|
25
|
+
},
|
|
26
|
+
};
|
|
27
|
+
const createStreetRule = (type, from = StreetNumberRule.MinFrom, to = StreetNumberRule.MaxTo) => {
|
|
28
|
+
return {
|
|
29
|
+
type: type,
|
|
30
|
+
from: from,
|
|
31
|
+
to: to,
|
|
32
|
+
};
|
|
33
|
+
};
|
|
34
|
+
const getStreetRules = (numberSpecification) => {
|
|
35
|
+
let rules = [];
|
|
36
|
+
let position = 0;
|
|
37
|
+
if (numberSpecification == "") {
|
|
38
|
+
return [StreetNumberRule.EmptyRule];
|
|
39
|
+
}
|
|
40
|
+
while (position < numberSpecification.length) {
|
|
41
|
+
var remainder = numberSpecification.substr(position);
|
|
42
|
+
var match = streetSeriesPartPattern.exec(remainder);
|
|
43
|
+
// if no series type is found, it's an error
|
|
44
|
+
if (match == null || match.index != 0) {
|
|
45
|
+
return null;
|
|
46
|
+
}
|
|
47
|
+
var type = getSeriesType(match[0]);
|
|
48
|
+
position += match[0].length;
|
|
49
|
+
remainder = numberSpecification.substr(position);
|
|
50
|
+
match = streetSeriesPartPattern.exec(remainder);
|
|
51
|
+
var end = numberSpecification.length;
|
|
52
|
+
if (match != null) {
|
|
53
|
+
end = position + match.index;
|
|
54
|
+
}
|
|
55
|
+
var rangePart = numberSpecification.substr(position, end - position);
|
|
56
|
+
rules = rules.concat(getRulesFromRangePart(rangePart, type));
|
|
57
|
+
position = end;
|
|
58
|
+
}
|
|
59
|
+
return rules;
|
|
60
|
+
};
|
|
61
|
+
const getRulesFromRangePart = (rangePart, seriesType) => {
|
|
62
|
+
let rules = [];
|
|
63
|
+
if (removeSeparatorFromEnd(rangePart) == "") {
|
|
64
|
+
rules.push(createStreetRule(seriesType));
|
|
65
|
+
return rules;
|
|
66
|
+
}
|
|
67
|
+
let position = 0;
|
|
68
|
+
rangePart = rangePart.trim() + ", ";
|
|
69
|
+
while (position < rangePart.length) {
|
|
70
|
+
var match = rangePattern.exec(rangePart.substr(position));
|
|
71
|
+
if (match == null) {
|
|
72
|
+
break;
|
|
73
|
+
}
|
|
74
|
+
position += match.index + match[0].length;
|
|
75
|
+
var value = removeSeparatorFromEnd(match[0]);
|
|
76
|
+
if (value.indexOf("-") != -1) {
|
|
77
|
+
var fromTo = value.split("-");
|
|
78
|
+
rules.push(createStreetRule(seriesType, parseInt(fromTo[0]), fromTo[1]));
|
|
79
|
+
}
|
|
80
|
+
else if (value.indexOf("výše") != -1) {
|
|
81
|
+
var number = parseInt(numberPattern.exec(value)[0]);
|
|
82
|
+
rules.push(createStreetRule(seriesType, number));
|
|
83
|
+
}
|
|
84
|
+
else {
|
|
85
|
+
var number = parseInt(numberPattern.exec(value)[0]);
|
|
86
|
+
rules.push(createStreetRule(seriesType, number, number));
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
return rules;
|
|
90
|
+
};
|
|
91
|
+
const getSeriesType = (pattern) => {
|
|
92
|
+
switch (pattern) {
|
|
93
|
+
case OddTypeString:
|
|
94
|
+
return StreetNumberSeriesType.Odd;
|
|
95
|
+
case EvenTypeString:
|
|
96
|
+
return StreetNumberSeriesType.Even;
|
|
97
|
+
case CPTypeString:
|
|
98
|
+
return StreetNumberSeriesType.CP;
|
|
99
|
+
case AllTypeString:
|
|
100
|
+
default:
|
|
101
|
+
return StreetNumberSeriesType.All;
|
|
102
|
+
}
|
|
103
|
+
};
|
|
104
|
+
const removeSeparatorFromEnd = (text) => {
|
|
105
|
+
text = text.trim();
|
|
106
|
+
if (text[text.length - 1] == "a" || text[text.length - 1] == ",") {
|
|
107
|
+
text = text.substr(0, text.length - 1);
|
|
108
|
+
}
|
|
109
|
+
return text.trim();
|
|
110
|
+
};
|
|
111
|
+
// desired output format:
|
|
112
|
+
// Příčná(str);from(int);to(int);[odd|even|all|cp];school(str)
|
|
113
|
+
export function districtsToCsvRules(districts, includeDistrictName = false) {
|
|
114
|
+
const lines = [];
|
|
115
|
+
districts.forEach((district) => {
|
|
116
|
+
district.schools.forEach((school) => {
|
|
117
|
+
school.lines.forEach((street) => {
|
|
118
|
+
if (street[0] == "!" || street == "") {
|
|
119
|
+
return;
|
|
120
|
+
}
|
|
121
|
+
let result = streetNamePattern.exec(street);
|
|
122
|
+
let streetName = result[0];
|
|
123
|
+
let numberSpecification = street.substr(streetName.length + 3).trim();
|
|
124
|
+
let rules = getStreetRules(numberSpecification);
|
|
125
|
+
if (rules == null || rules.length == 0) {
|
|
126
|
+
console.error("Error: " + street);
|
|
127
|
+
return;
|
|
128
|
+
}
|
|
129
|
+
rules.forEach((rule) => {
|
|
130
|
+
lines.push(streetName +
|
|
131
|
+
";" +
|
|
132
|
+
rule.from +
|
|
133
|
+
";" +
|
|
134
|
+
rule.to +
|
|
135
|
+
";" +
|
|
136
|
+
rule.type +
|
|
137
|
+
";" +
|
|
138
|
+
school.name +
|
|
139
|
+
(includeDistrictName ? ";" + district.name : ""));
|
|
140
|
+
});
|
|
141
|
+
});
|
|
142
|
+
});
|
|
143
|
+
});
|
|
144
|
+
return lines.join("\n");
|
|
145
|
+
}
|
|
146
|
+
// let raw = readFileSync("out.json", "utf-8");
|
|
147
|
+
// let districts;
|
|
148
|
+
// try {
|
|
149
|
+
// districts = JSON.parse(raw);
|
|
150
|
+
// } catch (error) {
|
|
151
|
+
// console.log(error);
|
|
152
|
+
// return;
|
|
153
|
+
// }
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
2
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
3
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
4
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
5
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
6
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
7
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
8
|
+
});
|
|
9
|
+
};
|
|
10
|
+
import { districtsToCsvRules } from "./json-to-csv.js";
|
|
11
|
+
import { getParsedDistricts } from "./txt-to-json.js";
|
|
12
|
+
import { writeFileSync } from "fs";
|
|
13
|
+
const main = () => __awaiter(void 0, void 0, void 0, function* () {
|
|
14
|
+
// take first node argument as a file name
|
|
15
|
+
if (process.argv.length < 3) {
|
|
16
|
+
console.error("Missing file name argument");
|
|
17
|
+
process.exit(1);
|
|
18
|
+
}
|
|
19
|
+
const fileName = process.argv[2];
|
|
20
|
+
const districts = yield getParsedDistricts(fileName);
|
|
21
|
+
const csv = districtsToCsvRules(districts);
|
|
22
|
+
if (process.argv.length >= 4) {
|
|
23
|
+
const outputFileName = process.argv[3];
|
|
24
|
+
console.log(`Writing output to ${outputFileName}`);
|
|
25
|
+
writeFileSync(outputFileName, csv);
|
|
26
|
+
}
|
|
27
|
+
else {
|
|
28
|
+
console.log(csv);
|
|
29
|
+
}
|
|
30
|
+
});
|
|
31
|
+
main();
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export function getParsedDistricts(filePath: any): Promise<any[]>;
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
2
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
3
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
4
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
5
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
6
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
7
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
8
|
+
});
|
|
9
|
+
};
|
|
10
|
+
import { createReadStream } from "fs";
|
|
11
|
+
import { once } from "events";
|
|
12
|
+
import { createInterface } from "readline";
|
|
13
|
+
const pattern = /^([^ –,-]+([ –-][^ ,–-]+)*)( [–-] (lichá č.|sudá č.|č.|č. p.)( (\d+[a-zA-Z]? ?[–-] ?\d+[a-zA-Z]?|(od )?\d+[a-zA-Z]?( a)? výše|\d+[a-zA-Z]?)((, ?| ?a ?)(\d+[a-zA-Z]? ?[–-] ?\d+[a-zA-Z]?|(od )?\d+[a-zA-Z]?( a)? výše|\d+[a-zA-Z]?))*)?((, ?| ?a ?)(lichá č.|sudá č.|č.|č. p.)( (\d+[a-zA-Z]? ?[–-] ?\d+[a-zA-Z]?|(od )?\d+[a-zA-Z]?( a)? výše|\d+[a-zA-Z]?)((, ?| ?a ?)(\d+[a-zA-Z]? ?[–-] ?\d+[a-zA-Z]?|(od )?\d+[a-zA-Z]?( a)? výše|\d+[a-zA-Z]?))*)?)*)?$/;
|
|
14
|
+
function getNewDistrict(name) {
|
|
15
|
+
return {
|
|
16
|
+
name: name,
|
|
17
|
+
schools: [],
|
|
18
|
+
};
|
|
19
|
+
}
|
|
20
|
+
function getNewSchool(name) {
|
|
21
|
+
return {
|
|
22
|
+
name: name,
|
|
23
|
+
lines: [],
|
|
24
|
+
};
|
|
25
|
+
}
|
|
26
|
+
export function getParsedDistricts(filePath) {
|
|
27
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
28
|
+
const rl = createInterface({
|
|
29
|
+
input: createReadStream(filePath),
|
|
30
|
+
});
|
|
31
|
+
let lineNumber = 1;
|
|
32
|
+
let districts = [];
|
|
33
|
+
let currentDistrict = null;
|
|
34
|
+
let currentSchool = null;
|
|
35
|
+
rl.on("line", function (line) {
|
|
36
|
+
let s = line
|
|
37
|
+
.trim()
|
|
38
|
+
.replace(/ +(?= )/g, "")
|
|
39
|
+
.replace(/–/g, "-")
|
|
40
|
+
.replace(/nábř\./g, "nábřeží")
|
|
41
|
+
.replace(/Nábř\./g, "Nábřeží")
|
|
42
|
+
.replace(/nám\./g, "náměstí")
|
|
43
|
+
.replace(/Nám\./g, "Náměstí");
|
|
44
|
+
if (s[0] == "#") {
|
|
45
|
+
if (currentSchool != null) {
|
|
46
|
+
if (currentDistrict == null) {
|
|
47
|
+
currentDistrict = getNewDistrict("");
|
|
48
|
+
}
|
|
49
|
+
currentDistrict.schools.push(currentSchool);
|
|
50
|
+
currentSchool = null;
|
|
51
|
+
}
|
|
52
|
+
if (currentDistrict != null) {
|
|
53
|
+
districts.push(currentDistrict);
|
|
54
|
+
}
|
|
55
|
+
currentDistrict = getNewDistrict(s.substring(1).trim());
|
|
56
|
+
currentSchool = null;
|
|
57
|
+
}
|
|
58
|
+
else if (s == "") {
|
|
59
|
+
if (currentSchool != null) {
|
|
60
|
+
if (currentDistrict == null) {
|
|
61
|
+
currentDistrict = getNewDistrict("");
|
|
62
|
+
}
|
|
63
|
+
currentDistrict.schools.push(currentSchool);
|
|
64
|
+
currentSchool = null;
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
else {
|
|
68
|
+
if (currentSchool == null) {
|
|
69
|
+
currentSchool = getNewSchool(s);
|
|
70
|
+
}
|
|
71
|
+
else {
|
|
72
|
+
if (s[0] == "!") {
|
|
73
|
+
currentSchool.lines.push(s);
|
|
74
|
+
}
|
|
75
|
+
else {
|
|
76
|
+
if (pattern.test(s)) {
|
|
77
|
+
currentSchool.lines.push(s);
|
|
78
|
+
}
|
|
79
|
+
else {
|
|
80
|
+
console.error("Invalid street line on line " + lineNumber + ": " + s);
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
lineNumber++;
|
|
86
|
+
});
|
|
87
|
+
rl.on("close", function () {
|
|
88
|
+
if (currentSchool != null) {
|
|
89
|
+
if (currentDistrict == null) {
|
|
90
|
+
currentDistrict = getNewDistrict("");
|
|
91
|
+
}
|
|
92
|
+
currentDistrict.schools.push(currentSchool);
|
|
93
|
+
}
|
|
94
|
+
if (currentDistrict != null) {
|
|
95
|
+
districts.push(currentDistrict);
|
|
96
|
+
}
|
|
97
|
+
// console.log(JSON.stringify(districts));
|
|
98
|
+
});
|
|
99
|
+
yield once(rl, "close");
|
|
100
|
+
return districts;
|
|
101
|
+
});
|
|
102
|
+
}
|