text-to-map 1.1.12 → 1.1.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -14,6 +14,7 @@ import iconv from "iconv-lite";
|
|
|
14
14
|
import fetch from "node-fetch";
|
|
15
15
|
import { join } from "path";
|
|
16
16
|
import { pipeline } from "stream/promises";
|
|
17
|
+
import chunk from "lodash/chunk";
|
|
17
18
|
import { commitAddressPoints } from "../db/address-points";
|
|
18
19
|
import { SyncPart } from "../db/types";
|
|
19
20
|
import { getLatestUrlFromAtomFeed } from "../utils/atom";
|
|
@@ -42,24 +43,21 @@ const importDataToDb = (options) => __awaiter(void 0, void 0, void 0, function*
|
|
|
42
43
|
let total = 0;
|
|
43
44
|
let next = 0;
|
|
44
45
|
console.log("Initiating import of RUIAN data to search DB (~3 million rows to be imported).");
|
|
45
|
-
const buffer = [];
|
|
46
46
|
for (const file of files) {
|
|
47
|
+
const rows = [];
|
|
47
48
|
const parseStream = parse({ delimiter: ";", fromLine: 2 }).on("data", (data) => __awaiter(void 0, void 0, void 0, function* () {
|
|
48
|
-
|
|
49
|
-
if (buffer.length >= maxBufferSize) {
|
|
50
|
-
parseStream.pause();
|
|
51
|
-
total += yield commitAddressPoints(buffer);
|
|
52
|
-
if (total - next >= 100000) {
|
|
53
|
-
next += 100000;
|
|
54
|
-
console.log(`Total imported rows: ${next}`);
|
|
55
|
-
}
|
|
56
|
-
buffer.length = 0;
|
|
57
|
-
parseStream.resume();
|
|
58
|
-
}
|
|
49
|
+
rows.push(data);
|
|
59
50
|
}));
|
|
60
51
|
yield pipeline(createReadStream(join(extractionFolder, file)), iconv.decodeStream("win1250"), parseStream);
|
|
52
|
+
const chunks = chunk(rows, maxBufferSize);
|
|
53
|
+
yield Promise.all(chunks.map((chunk) => __awaiter(void 0, void 0, void 0, function* () {
|
|
54
|
+
total += yield commitAddressPoints(chunk);
|
|
55
|
+
if (total - next >= 100000) {
|
|
56
|
+
next += 100000;
|
|
57
|
+
console.log(`Total imported rows: ${next}`);
|
|
58
|
+
}
|
|
59
|
+
})));
|
|
61
60
|
}
|
|
62
|
-
total += yield commitAddressPoints(buffer);
|
|
63
61
|
console.log(`Import completed. Total imported rows: ${total}`);
|
|
64
62
|
});
|
|
65
63
|
const getExtractionFolder = (options) => join(options.tmpDir, options.addressPointsCsvFolderName);
|
package/package.json
CHANGED
|
@@ -11,6 +11,7 @@
|
|
|
11
11
|
"feedparser": "^2.2.10",
|
|
12
12
|
"iconv-lite": "^0.6.3",
|
|
13
13
|
"knex": "^2.4.2",
|
|
14
|
+
"lodash": "^4.17.21",
|
|
14
15
|
"mysql": "^2.18.1",
|
|
15
16
|
"node-fetch": "^3.3.0",
|
|
16
17
|
"parsedbf": "^1.1.1",
|
|
@@ -24,6 +25,7 @@
|
|
|
24
25
|
"@types/adm-zip": "^0.5.0",
|
|
25
26
|
"@types/better-sqlite3": "^7.6.3",
|
|
26
27
|
"@types/feedparser": "^2.2.5",
|
|
28
|
+
"@types/lodash": "^4.14.196",
|
|
27
29
|
"@types/node": "^18.11.17",
|
|
28
30
|
"copyfiles": "^2.4.1",
|
|
29
31
|
"jest": "^29.5.0",
|
|
@@ -54,7 +56,7 @@
|
|
|
54
56
|
},
|
|
55
57
|
"name": "text-to-map",
|
|
56
58
|
"description": "Text To Map usiluje o lepší, strojově zpracovatelné využití částí vyhlášek s výčtem ulic a dalších lokací. Jde o rozšiřitelnou sadu konceptů a nástrojů, které zajistí hladký převod výčtu ulic a jejich rozsahů v lidsky srozumitelném jazyce do strojově zpracovatelného, uchopitelného formátu.",
|
|
57
|
-
"version": "1.1.
|
|
59
|
+
"version": "1.1.13",
|
|
58
60
|
"main": "dist/index.js",
|
|
59
61
|
"types": "dist/index.d.ts",
|
|
60
62
|
"files": [
|