posti 3.5.1 → 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/{LICENSE.md → LICENSE} +1 -1
- package/README.md +333 -112
- package/dist/cli/args.d.ts +17 -0
- package/dist/cli/args.d.ts.map +1 -0
- package/dist/cli/args.js +80 -0
- package/dist/cli/args.js.map +1 -0
- package/dist/cli.d.ts +3 -0
- package/dist/cli.d.ts.map +1 -0
- package/dist/cli.js +85 -0
- package/dist/cli.js.map +1 -0
- package/dist/config/defaults.d.ts +2 -0
- package/dist/config/defaults.d.ts.map +1 -0
- package/dist/config/defaults.js +2 -0
- package/dist/config/defaults.js.map +1 -0
- package/dist/config/index.d.ts +5 -0
- package/dist/config/index.d.ts.map +1 -0
- package/dist/config/index.js +97 -1
- package/dist/config/index.js.map +1 -0
- package/dist/config/paths.d.ts +4 -0
- package/dist/config/paths.d.ts.map +1 -0
- package/dist/config/paths.js +16 -0
- package/dist/config/paths.js.map +1 -0
- package/dist/config/schema.d.ts +27 -0
- package/dist/config/schema.d.ts.map +1 -0
- package/dist/config/schema.js +24 -0
- package/dist/config/schema.js.map +1 -0
- package/dist/db/dialects.d.ts +15 -0
- package/dist/db/dialects.d.ts.map +1 -0
- package/dist/db/dialects.js +270 -0
- package/dist/db/dialects.js.map +1 -0
- package/dist/db/index.d.ts +27 -0
- package/dist/db/index.d.ts.map +1 -0
- package/dist/db/index.js +49 -0
- package/dist/db/index.js.map +1 -0
- package/dist/db/migrate.d.ts +8 -0
- package/dist/db/migrate.d.ts.map +1 -0
- package/dist/db/migrate.js +58 -0
- package/dist/db/migrate.js.map +1 -0
- package/dist/db/migrations/001_initial_schema.d.ts +3 -0
- package/dist/db/migrations/001_initial_schema.d.ts.map +1 -0
- package/dist/db/migrations/001_initial_schema.js +17 -0
- package/dist/db/migrations/001_initial_schema.js.map +1 -0
- package/dist/db/migrations/index.d.ts +3 -0
- package/dist/db/migrations/index.d.ts.map +1 -0
- package/dist/db/migrations/index.js +3 -0
- package/dist/db/migrations/index.js.map +1 -0
- package/dist/db/operations.d.ts +13 -0
- package/dist/db/operations.d.ts.map +1 -0
- package/dist/db/operations.js +179 -0
- package/dist/db/operations.js.map +1 -0
- package/dist/fetcher/url-parser.d.ts +9 -0
- package/dist/fetcher/url-parser.d.ts.map +1 -0
- package/dist/fetcher/url-parser.js +19 -0
- package/dist/fetcher/url-parser.js.map +1 -0
- package/dist/index.d.ts +15 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +9 -1
- package/dist/index.js.map +1 -0
- package/dist/loader/index.d.ts +7 -0
- package/dist/loader/index.d.ts.map +1 -0
- package/dist/loader/index.js +145 -0
- package/dist/loader/index.js.map +1 -0
- package/dist/loader/state.d.ts +5 -0
- package/dist/loader/state.d.ts.map +1 -0
- package/dist/loader/state.js +26 -0
- package/dist/loader/state.js.map +1 -0
- package/dist/parser/field-definitions.d.ts +14 -0
- package/dist/parser/field-definitions.d.ts.map +1 -0
- package/dist/parser/field-definitions.js +91 -0
- package/dist/parser/field-definitions.js.map +1 -0
- package/dist/parser/file-converter.d.ts +2 -0
- package/dist/parser/file-converter.d.ts.map +1 -0
- package/dist/parser/file-converter.js +7 -0
- package/dist/parser/file-converter.js.map +1 -0
- package/dist/parser/line-parser.d.ts +4 -0
- package/dist/parser/line-parser.d.ts.map +1 -0
- package/dist/parser/line-parser.js +28 -0
- package/dist/parser/line-parser.js.map +1 -0
- package/dist/parser/unzipper.d.ts +2 -0
- package/dist/parser/unzipper.d.ts.map +1 -0
- package/dist/parser/unzipper.js +11 -0
- package/dist/parser/unzipper.js.map +1 -0
- package/dist/utils/chunks.d.ts +2 -0
- package/dist/utils/chunks.d.ts.map +1 -0
- package/dist/utils/chunks.js +10 -0
- package/dist/utils/chunks.js.map +1 -0
- package/dist/utils/logger.d.ts +7 -0
- package/dist/utils/logger.d.ts.map +1 -0
- package/dist/utils/logger.js +8 -0
- package/dist/utils/logger.js.map +1 -0
- package/dist/utils/progress.d.ts +14 -0
- package/dist/utils/progress.d.ts.map +1 -0
- package/dist/utils/progress.js +68 -0
- package/dist/utils/progress.js.map +1 -0
- package/dist/utils/time.d.ts +2 -0
- package/dist/utils/time.d.ts.map +1 -0
- package/dist/utils/time.js +14 -0
- package/dist/utils/time.js.map +1 -0
- package/package.json +76 -63
- package/CHANGELOG.md +0 -351
- package/dist/classes/Posti.js +0 -11
- package/dist/classes/Sequelize.js +0 -15
- package/dist/posti.js +0 -2
- package/dist/utils/index.js +0 -55
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
import { createReadStream } from "node:fs";
|
|
2
|
+
import fs from "node:fs/promises";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
import { createInterface } from "node:readline";
|
|
5
|
+
import { POSTI_INDEX_URL } from "../config/defaults.js";
|
|
6
|
+
import { getCacheDir, getStateDir } from "../config/paths.js";
|
|
7
|
+
import { closeConnection, createConnection } from "../db/index.js";
|
|
8
|
+
import { runMigrations } from "../db/migrate.js";
|
|
9
|
+
import { createTempTable, dropTable, insertRows, upsertFromTemp } from "../db/operations.js";
|
|
10
|
+
import { parseFileUrls } from "../fetcher/url-parser.js";
|
|
11
|
+
import { fileTypeToFields, fileTypeToTable } from "../parser/field-definitions.js";
|
|
12
|
+
import { convertFile } from "../parser/file-converter.js";
|
|
13
|
+
import { parseLine } from "../parser/line-parser.js";
|
|
14
|
+
import { extractZip } from "../parser/unzipper.js";
|
|
15
|
+
import { sliceIntoChunks } from "../utils/chunks.js";
|
|
16
|
+
import { logger } from "../utils/logger.js";
|
|
17
|
+
import { createProgress } from "../utils/progress.js";
|
|
18
|
+
import { getNewFiles, writeLatest } from "./state.js";
|
|
19
|
+
export async function run(config, options = {}) {
|
|
20
|
+
const cacheDir = getCacheDir();
|
|
21
|
+
const stateDir = getStateDir();
|
|
22
|
+
const stateFile = path.join(stateDir, "latest.json");
|
|
23
|
+
// Ensure directories exist
|
|
24
|
+
await fs.mkdir(cacheDir, { recursive: true });
|
|
25
|
+
await fs.mkdir(stateDir, { recursive: true });
|
|
26
|
+
// Fetch file index
|
|
27
|
+
logger.info("Fetching file list from Posti...");
|
|
28
|
+
const response = await fetch(POSTI_INDEX_URL);
|
|
29
|
+
if (!response.ok) {
|
|
30
|
+
throw new Error(`Failed to fetch file list from ${POSTI_INDEX_URL}: HTTP ${response.status}`);
|
|
31
|
+
}
|
|
32
|
+
const html = await response.text();
|
|
33
|
+
const allFiles = parseFileUrls(html);
|
|
34
|
+
logger.info(`Found ${allFiles.length} files`);
|
|
35
|
+
// Filter to configured tables
|
|
36
|
+
const files = allFiles.filter((f) => {
|
|
37
|
+
const table = fileTypeToTable[f.fileType];
|
|
38
|
+
return table && config.tables.includes(table);
|
|
39
|
+
});
|
|
40
|
+
// Check for new files
|
|
41
|
+
const newFiles = await getNewFiles(files, stateFile, options.force);
|
|
42
|
+
if (newFiles.length === 0) {
|
|
43
|
+
logger.info("No new files to process");
|
|
44
|
+
return;
|
|
45
|
+
}
|
|
46
|
+
logger.info(`Processing ${newFiles.length} new file(s)`);
|
|
47
|
+
const progress = createProgress({ silent: options.silent });
|
|
48
|
+
// Connect to database
|
|
49
|
+
const db = await createConnection(config);
|
|
50
|
+
try {
|
|
51
|
+
// Run pending migrations (unless migrations are disabled)
|
|
52
|
+
if (config.migrate) {
|
|
53
|
+
await runMigrations(db, config);
|
|
54
|
+
}
|
|
55
|
+
progress.startOverall(newFiles.length);
|
|
56
|
+
// Process each file
|
|
57
|
+
for (const file of newFiles) {
|
|
58
|
+
await processFile(db, config, file, cacheDir, progress);
|
|
59
|
+
}
|
|
60
|
+
progress.stop();
|
|
61
|
+
// Update state
|
|
62
|
+
await writeLatest(stateFile, files.map((f) => f.filename));
|
|
63
|
+
logger.info("Done");
|
|
64
|
+
}
|
|
65
|
+
finally {
|
|
66
|
+
await closeConnection(db);
|
|
67
|
+
// Clean up cache if configured
|
|
68
|
+
if (config.process.deleteOnComplete) {
|
|
69
|
+
await fs.rm(cacheDir, { recursive: true, force: true });
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
async function processFile(db, config, file, cacheDir, progress) {
|
|
74
|
+
const tableType = fileTypeToTable[file.fileType];
|
|
75
|
+
if (!tableType)
|
|
76
|
+
return;
|
|
77
|
+
const fields = fileTypeToFields[file.fileType];
|
|
78
|
+
if (!fields)
|
|
79
|
+
return;
|
|
80
|
+
progress.startFile(file.filename);
|
|
81
|
+
// Download
|
|
82
|
+
progress.updateFileStep("Downloading");
|
|
83
|
+
const zipPath = path.join(cacheDir, file.filename);
|
|
84
|
+
await downloadFile(file.url, zipPath);
|
|
85
|
+
// Unzip
|
|
86
|
+
progress.updateFileStep("Extracting");
|
|
87
|
+
await extractZip(zipPath, cacheDir);
|
|
88
|
+
// Convert encoding
|
|
89
|
+
progress.updateFileStep("Converting encoding");
|
|
90
|
+
const datPath = path.join(cacheDir, `${file.extensionless}.dat`);
|
|
91
|
+
const utf8Path = path.join(cacheDir, `${file.extensionless}_utf8.dat`);
|
|
92
|
+
await convertFile(datPath, utf8Path);
|
|
93
|
+
// Parse lines
|
|
94
|
+
progress.updateFileStep("Parsing");
|
|
95
|
+
const rows = await parseDataFile(utf8Path, fields);
|
|
96
|
+
// Load into database using temp table + upsert
|
|
97
|
+
const finalTable = `${config.tablePrefix}${tableType}`;
|
|
98
|
+
const tempTable = `${config.tablePrefix}temp_${tableType}`;
|
|
99
|
+
await createTempTable(db, tempTable, finalTable);
|
|
100
|
+
// Bulk insert into temp table in chunks
|
|
101
|
+
const chunks = sliceIntoChunks(rows, config.process.chunkSize);
|
|
102
|
+
let inserted = 0;
|
|
103
|
+
for (const chunk of chunks) {
|
|
104
|
+
await insertRows(db, tempTable, chunk);
|
|
105
|
+
inserted += chunk.length;
|
|
106
|
+
progress.updateFileProgress("Loading", inserted, rows.length);
|
|
107
|
+
}
|
|
108
|
+
// Upsert from temp to final
|
|
109
|
+
progress.updateFileStep("Upserting");
|
|
110
|
+
await upsertFromTemp(db, config.dialect, tempTable, finalTable, tableType);
|
|
111
|
+
// Drop temp table
|
|
112
|
+
await dropTable(db, tempTable);
|
|
113
|
+
progress.completeFile();
|
|
114
|
+
}
|
|
115
|
+
async function downloadFile(url, destPath) {
|
|
116
|
+
logger.info(" Downloading...");
|
|
117
|
+
const response = await fetch(url);
|
|
118
|
+
if (!response.ok || !response.body) {
|
|
119
|
+
throw new Error(`Download failed: ${url} (${response.status})`);
|
|
120
|
+
}
|
|
121
|
+
const arrayBuffer = await response.arrayBuffer();
|
|
122
|
+
await fs.writeFile(destPath, Buffer.from(arrayBuffer));
|
|
123
|
+
}
|
|
124
|
+
async function parseDataFile(filePath, fields) {
|
|
125
|
+
const rows = [];
|
|
126
|
+
const rl = createInterface({
|
|
127
|
+
input: createReadStream(filePath, "utf-8"),
|
|
128
|
+
crlfDelay: Infinity,
|
|
129
|
+
});
|
|
130
|
+
for await (const line of rl) {
|
|
131
|
+
if (line.trim()) {
|
|
132
|
+
// Use column names (snake_case) as keys for database insertion
|
|
133
|
+
const parsed = parseLine(line, fields);
|
|
134
|
+
const row = {};
|
|
135
|
+
for (const field of fields) {
|
|
136
|
+
if (field.start !== undefined) {
|
|
137
|
+
row[field.column] = parsed[field.name] ?? null;
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
rows.push(row);
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
return rows;
|
|
144
|
+
}
|
|
145
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/loader/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAC3C,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAClC,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAChD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAG9D,OAAO,EAAE,eAAe,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AACnE,OAAO,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AACjD,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAE7F,OAAO,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AAEzD,OAAO,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,gCAAgC,CAAC;AACnF,OAAO,EAAE,WAAW,EAAE,MAAM,6BAA6B,CAAC;AAC1D,OAAO,EAAE,SAAS,EAAE,MAAM,0BAA0B,CAAC;AACrD,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AACnD,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAC5C,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAOtD,MAAM,CAAC,KAAK,UAAU,GAAG,CAAC,MAAmB,EAAE,OAAO,GAAe,EAAE;IACrE,MAAM,QAAQ,GAAG,WAAW,EAAE,CAAC;IAC/B,MAAM,QAAQ,GAAG,WAAW,EAAE,CAAC;IAC/B,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC;IAErD,2BAA2B;IAC3B,MAAM,EAAE,CAAC,KAAK,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC9C,MAAM,EAAE,CAAC,KAAK,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAE9C,mBAAmB;IACnB,MAAM,CAAC,IAAI,CAAC,kCAAkC,CAAC,CAAC;IAChD,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,eAAe,CAAC,CAAC;IAC9C,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;QACjB,MAAM,IAAI,KAAK,CAAC,kCAAkC,eAAe,UAAU,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;IAChG,CAAC;IACD,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;IACnC,MAAM,QAAQ,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC;IAErC,MAAM,CAAC,IAAI,CAAC,SAAS,QAAQ,CAAC,MAAM,QAAQ,CAAC,CAAC;IAE9C,8BAA8B;IAC9B,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;QAClC,MAAM,KAAK,GAAG,eAAe,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;QAC1C,OAAO,KAAK,IAAI,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAuC,CAAC,CAAC;IAClF,CAAC,CAAC,CAAC;IAEH,sBAAsB;IACtB,MAAM,QAAQ,GAAG,MAAM,WAAW,CAAC,KAAK,EAAE,SAAS,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;IAEpE,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,MAAM,CAAC,IAAI,CAAC,yBAAyB,CAAC,CAAC;QACvC,OAAO;IACT,CAAC;IAED,MAAM,CAAC,IAAI,CAAC,cAAc,QAAQ,CAAC,MAAM,cAAc,CAAC,CAAC;IAEzD,MAAM,QAAQ,GAAG,cAAc,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAE5D,sBAAsB;IACtB,MAAM,EAAE,GAAG,MAAM,gBAAgB,CAAC,MAAM,CAAC,CAAC;IAE1C,IAAI,CAAC;QACH,0DAA0D;QAC1D,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YACnB,MAAM,aAAa,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;QAClC,CAAC;QAED,QAAQ,CAAC,YAAY,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;QAEvC,oBAAoB;QACpB,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;YAC5B,MAAM,WAAW,CAAC,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;QAC1D,CAAC;QAED,QAAQ,CAAC,IAAI,EAAE,CAAC;QAEhB,eAAe;QACf,MAAM,WAAW,CACf,SAAS,EACT,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAC7B,CAAC;QACF,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACtB,CAAC;YAAS,CAAC;QACT,MAAM,eAAe,CAAC,EAAE,CAAC,CAAC;QAE1B,+BAA+B;QAC/B,IAAI,MAAM,CAAC,OAAO,CAAC,gBAAgB,EAAE,CAAC;YACpC,MAAM,EAAE,CAAC,EAAE,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QAC1D,CAAC;IACH,CAAC;AACH,CAAC;AAED,KAAK,UAAU,WAAW,CACxB,EAAgB,EAChB,MAAmB,EACnB,IAAe,EACf,QAAgB,EAChB,QAA2C;IAE3C,MAAM,SAAS,GAAG,eAAe,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACjD,IAAI,CAAC,SAAS;QAAE,OAAO;IAEvB,MAAM,MAAM,GAAG,gBAAgB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAC/C,IAAI,CAAC,MAAM;QAAE,OAAO;IAEpB,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAElC,WAAW;IACX,QAAQ,CAAC,cAAc,CAAC,aAAa,CAAC,CAAC;IACvC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;IACnD,MAAM,YAAY,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;IAEtC,QAAQ;IACR,QAAQ,CAAC,cAAc,CAAC,YAAY,CAAC,CAAC;IACtC,MAAM,UAAU,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;IAEpC,mBAAmB;IACnB,QAAQ,CAAC,cAAc,CAAC,qBAAqB,CAAC,CAAC;IAC/C,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,GAAG,IAAI,CAAC,aAAa,MAAM,CAAC,CAAC;IACjE,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,GAAG,IAAI,CAAC,aAAa,WAAW,CAAC,CAAC;IACvE,MAAM,WAAW,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;IAErC,cAAc;IACd,QAAQ,CAAC,cAAc,CAAC,SAAS,CAAC,CAAC;IACnC,MAAM,IAAI,GAAG,MAAM,aAAa,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IAEnD,+CAA+C;IAC/C,MAAM,UAAU,GAAG,GAAG,MAAM,CAAC,WAAW,GAAG,SAAS,EAAE,CAAC;IACvD,MAAM,SAAS,GAAG,GAAG,MAAM,CAAC,WAAW,QAAQ,SAAS,EAAE,CAAC;IAE3D,MAAM,eAAe,CAAC,EAAE,EAAE,SAAS,EAAE,UAAU,CAAC,CAAC;IAEjD,wCAAwC;IACxC,MAAM,MAAM,GAAG,eAAe,CAAC,IAAI,EAAE,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IAC/D,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,MAAM,UAAU,CAAC,EAAE,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC;QACvC,QAAQ,IAAI,KAAK,CAAC,MAAM,CAAC;QACzB,QAAQ,CAAC,kBAAkB,CAAC,SAAS,EAAE,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;IAChE,CAAC;IAED,4BAA4B;IAC5B,QAAQ,CAAC,cAAc,CAAC,WAAW,CAAC,CAAC;IACrC,MAAM,cAAc,CAAC,EAAE,EAAE,MAAM,CAAC,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,SAAS,CAAC,CAAC;IAE3E,kBAAkB;IAClB,MAAM,SAAS,CAAC,EAAE,EAAE,SAAS,CAAC,CAAC;IAE/B,QAAQ,CAAC,YAAY,EAAE,CAAC;AAC1B,CAAC;AAED,KAAK,UAAU,YAAY,CAAC,GAAW,EAAE,QAAgB;IACvD,MAAM,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;IAChC,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,CAAC;IAClC,IAAI,CAAC,QAAQ,CAAC,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;QACnC,MAAM,IAAI,KAAK,CAAC,oBAAoB,GAAG,KAAK,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC;IAClE,CAAC;IAED,MAAM,WAAW,GAAG,MAAM,QAAQ,CAAC,WAAW,EAAE,CAAC;IACjD,MAAM,EAAE,CAAC,SAAS,CAAC,QAAQ,EAAE,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC;AACzD,CAAC;AAED,KAAK,UAAU,aAAa,CAC1B,QAAgB,EAChB,MAAyB;IAEzB,MAAM,IAAI,GAA6C,EAAE,CAAC;IAE1D,MAAM,EAAE,GAAG,eAAe,CAAC;QACzB,KAAK,EAAE,gBAAgB,CAAC,QAAQ,EAAE,OAAO,CAAC;QAC1C,SAAS,EAAE,QAAQ;KACpB,CAAC,CAAC;IAEH,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,EAAE,EAAE,CAAC;QAC5B,IAAI,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;YAChB,+DAA+D;YAC/D,MAAM,MAAM,GAAG,SAAS,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YACvC,MAAM,GAAG,GAA2C,EAAE,CAAC;YACvD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;gBAC3B,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;oBAC9B,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC;gBACjD,CAAC;YACH,CAAC;YACD,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACjB,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC"}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
import type { PostiFile } from "../fetcher/url-parser.js";
|
|
2
|
+
export declare function readLatest(stateFile: string): Promise<string[]>;
|
|
3
|
+
export declare function writeLatest(stateFile: string, filenames: string[]): Promise<void>;
|
|
4
|
+
export declare function getNewFiles(files: PostiFile[], stateFile: string, force?: boolean): Promise<PostiFile[]>;
|
|
5
|
+
//# sourceMappingURL=state.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"state.d.ts","sourceRoot":"","sources":["../../src/loader/state.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,0BAA0B,CAAC;AAE1D,wBAAsB,UAAU,CAAC,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAWrE;AAED,wBAAsB,WAAW,CAAC,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAGvF;AAED,wBAAsB,WAAW,CAAC,KAAK,EAAE,SAAS,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,KAAK,UAAQ,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC,CAK5G"}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import fs from "node:fs/promises";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
export async function readLatest(stateFile) {
|
|
4
|
+
try {
|
|
5
|
+
const content = await fs.readFile(stateFile, "utf-8");
|
|
6
|
+
const parsed = JSON.parse(content);
|
|
7
|
+
if (!Array.isArray(parsed) || !parsed.every((item) => typeof item === "string")) {
|
|
8
|
+
return [];
|
|
9
|
+
}
|
|
10
|
+
return parsed;
|
|
11
|
+
}
|
|
12
|
+
catch {
|
|
13
|
+
return [];
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
export async function writeLatest(stateFile, filenames) {
|
|
17
|
+
await fs.mkdir(path.dirname(stateFile), { recursive: true });
|
|
18
|
+
await fs.writeFile(stateFile, JSON.stringify(filenames), "utf-8");
|
|
19
|
+
}
|
|
20
|
+
export async function getNewFiles(files, stateFile, force = false) {
|
|
21
|
+
if (force)
|
|
22
|
+
return files;
|
|
23
|
+
const latest = await readLatest(stateFile);
|
|
24
|
+
return files.filter((file) => !latest.includes(file.filename));
|
|
25
|
+
}
|
|
26
|
+
//# sourceMappingURL=state.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"state.js","sourceRoot":"","sources":["../../src/loader/state.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAClC,OAAO,IAAI,MAAM,WAAW,CAAC;AAG7B,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,SAAiB;IAChD,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;QACtD,MAAM,MAAM,GAAY,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC5C,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,IAAI,KAAK,QAAQ,CAAC,EAAE,CAAC;YAChF,OAAO,EAAE,CAAC;QACZ,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,SAAiB,EAAE,SAAmB;IACtE,MAAM,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC7D,MAAM,EAAE,CAAC,SAAS,CAAC,SAAS,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,EAAE,OAAO,CAAC,CAAC;AACpE,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,KAAkB,EAAE,SAAiB,EAAE,KAAK,GAAG,KAAK;IACpF,IAAI,KAAK;QAAE,OAAO,KAAK,CAAC;IAExB,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,SAAS,CAAC,CAAC;IAC3C,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;AACjE,CAAC"}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
export interface FieldDefinition {
|
|
2
|
+
name: string;
|
|
3
|
+
column: string;
|
|
4
|
+
start?: number;
|
|
5
|
+
length: number;
|
|
6
|
+
type: "string" | "integer" | "date";
|
|
7
|
+
}
|
|
8
|
+
export type FileType = "BAF" | "PCF" | "POM";
|
|
9
|
+
export declare const addressFields: FieldDefinition[];
|
|
10
|
+
export declare const zipcodeFields: FieldDefinition[];
|
|
11
|
+
export declare const zipcodeChangeFields: FieldDefinition[];
|
|
12
|
+
export declare const fileTypeToFields: Record<FileType, FieldDefinition[]>;
|
|
13
|
+
export declare const fileTypeToTable: Record<FileType, string>;
|
|
14
|
+
//# sourceMappingURL=field-definitions.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"field-definitions.d.ts","sourceRoot":"","sources":["../../src/parser/field-definitions.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,QAAQ,GAAG,SAAS,GAAG,MAAM,CAAC;CACrC;AAED,MAAM,MAAM,QAAQ,GAAG,KAAK,GAAG,KAAK,GAAG,KAAK,CAAC;AAG7C,eAAO,MAAM,aAAa,EAAE,eAAe,EA0B1C,CAAC;AAGF,eAAO,MAAM,aAAa,EAAE,eAAe,EAiB1C,CAAC;AAGF,eAAO,MAAM,mBAAmB,EAAE,eAAe,EA+BhD,CAAC;AAEF,eAAO,MAAM,gBAAgB,EAAE,MAAM,CAAC,QAAQ,EAAE,eAAe,EAAE,CAIhE,CAAC;AAEF,eAAO,MAAM,eAAe,EAAE,MAAM,CAAC,QAAQ,EAAE,MAAM,CAIpD,CAAC"}
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
// BAF — address file (fixed-width, 257 chars/line)
|
|
2
|
+
export const addressFields = [
|
|
3
|
+
{ name: "record", column: "record", start: 1, length: 5, type: "string" },
|
|
4
|
+
{ name: "updatedAt", column: "updated_at", start: 6, length: 8, type: "date" },
|
|
5
|
+
{ name: "postalCode", column: "postal_code", start: 14, length: 5, type: "string" },
|
|
6
|
+
{ name: "postOfficeName", column: "post_office_name", start: 19, length: 30, type: "string" },
|
|
7
|
+
{ name: "postOfficeNameSwe", column: "post_office_name_swe", start: 49, length: 30, type: "string" },
|
|
8
|
+
{ name: "postOfficeShortName", column: "post_office_short_name", start: 79, length: 12, type: "string" },
|
|
9
|
+
{ name: "postOfficeShortNameSwe", column: "post_office_short_name_swe", start: 91, length: 12, type: "string" },
|
|
10
|
+
{ name: "address", column: "address", start: 103, length: 30, type: "string" },
|
|
11
|
+
{ name: "addressSwe", column: "address_swe", start: 133, length: 30, type: "string" },
|
|
12
|
+
{ name: "empty1", column: "empty1", start: 163, length: 12, type: "string" },
|
|
13
|
+
{ name: "empty2", column: "empty2", start: 175, length: 12, type: "string" },
|
|
14
|
+
{ name: "oddEven", column: "odd_even", start: 187, length: 1, type: "integer" },
|
|
15
|
+
{ name: "smallestBuildingNumber1", column: "smallest_building_number_1", start: 188, length: 5, type: "integer" },
|
|
16
|
+
{ name: "smallestDeliveryLetter1", column: "smallest_delivery_letter_1", start: 193, length: 1, type: "string" },
|
|
17
|
+
{ name: "divider1", column: "divider_1", start: 194, length: 1, type: "string" },
|
|
18
|
+
{ name: "smallestBuildingNumber2", column: "smallest_building_number_2", start: 195, length: 5, type: "integer" },
|
|
19
|
+
{ name: "smallestDeliveryLetter2", column: "smallest_delivery_letter_2", start: 200, length: 1, type: "string" },
|
|
20
|
+
{ name: "highestBuildingNumber1", column: "highest_building_number_1", start: 201, length: 5, type: "integer" },
|
|
21
|
+
{ name: "highestDeliveryLetter1", column: "highest_delivery_letter_1", start: 206, length: 1, type: "string" },
|
|
22
|
+
{ name: "divider2", column: "divider_2", start: 207, length: 1, type: "string" },
|
|
23
|
+
{ name: "highestBuildingNumber2", column: "highest_building_number_2", start: 208, length: 5, type: "integer" },
|
|
24
|
+
{ name: "highestDeliveryLetter2", column: "highest_delivery_letter_2", start: 213, length: 1, type: "string" },
|
|
25
|
+
{ name: "municipalityIdCode", column: "municipality_id_code", start: 214, length: 3, type: "string" },
|
|
26
|
+
{ name: "municipalityName", column: "municipality_name", start: 217, length: 20, type: "string" },
|
|
27
|
+
{ name: "municipalityNameSwe", column: "municipality_name_swe", start: 237, length: 20, type: "string" },
|
|
28
|
+
];
|
|
29
|
+
// PCF — postal code file (fixed-width, 221 chars/line)
|
|
30
|
+
export const zipcodeFields = [
|
|
31
|
+
{ name: "record", column: "record", start: 1, length: 5, type: "string" },
|
|
32
|
+
{ name: "updatedAt", column: "updated_at", start: 6, length: 8, type: "date" },
|
|
33
|
+
{ name: "postalCode", column: "postal_code", start: 14, length: 5, type: "string" },
|
|
34
|
+
{ name: "postOfficeName", column: "post_office_name", start: 19, length: 30, type: "string" },
|
|
35
|
+
{ name: "postOfficeNameSwe", column: "post_office_name_swe", start: 49, length: 30, type: "string" },
|
|
36
|
+
{ name: "postOfficeShortName", column: "post_office_short_name", start: 79, length: 12, type: "string" },
|
|
37
|
+
{ name: "postOfficeShortNameSwe", column: "post_office_short_name_swe", start: 91, length: 12, type: "string" },
|
|
38
|
+
{ name: "entryIntoForceAt", column: "entry_into_force_at", start: 103, length: 8, type: "date" },
|
|
39
|
+
{ name: "typeCode", column: "type_code", start: 111, length: 1, type: "integer" },
|
|
40
|
+
{ name: "regionId", column: "region_id", start: 112, length: 5, type: "string" },
|
|
41
|
+
{ name: "regionName", column: "region_name", start: 117, length: 30, type: "string" },
|
|
42
|
+
{ name: "regionNameSwe", column: "region_name_swe", start: 147, length: 30, type: "string" },
|
|
43
|
+
{ name: "municipalityIdCode", column: "municipality_id_code", start: 177, length: 3, type: "string" },
|
|
44
|
+
{ name: "municipalityName", column: "municipality_name", start: 180, length: 20, type: "string" },
|
|
45
|
+
{ name: "municipalityNameSwe", column: "municipality_name_swe", start: 200, length: 20, type: "string" },
|
|
46
|
+
{ name: "municipalityLanguage", column: "municipality_language", start: 220, length: 1, type: "integer" },
|
|
47
|
+
];
|
|
48
|
+
// POM — postal code change file (fixed-width, 453 chars/line)
|
|
49
|
+
export const zipcodeChangeFields = [
|
|
50
|
+
{ name: "record", column: "record", start: 1, length: 4, type: "string" },
|
|
51
|
+
{ name: "level", column: "level", start: 5, length: 1, type: "string" },
|
|
52
|
+
{ name: "updatedAt", column: "updated_at", start: 6, length: 8, type: "date" },
|
|
53
|
+
{ name: "extractionStartAt", column: "extraction_start_at", start: 14, length: 8, type: "date" },
|
|
54
|
+
{ name: "extractionEndAt", column: "extraction_end_at", start: 22, length: 8, type: "date" },
|
|
55
|
+
{ name: "oldPostalCode", column: "old_postal_code", start: 30, length: 5, type: "string" },
|
|
56
|
+
{ name: "oldPostOfficeName", column: "old_post_office_name", start: 35, length: 30, type: "string" },
|
|
57
|
+
{ name: "oldPostOfficeNameSwe", column: "old_post_office_name_swe", start: 65, length: 30, type: "string" },
|
|
58
|
+
{ name: "oldPostOfficeShortName", column: "old_post_office_short_name", start: 95, length: 12, type: "string" },
|
|
59
|
+
{
|
|
60
|
+
name: "oldPostOfficeShortNameSwe",
|
|
61
|
+
column: "old_post_office_short_name_swe",
|
|
62
|
+
start: 107,
|
|
63
|
+
length: 12,
|
|
64
|
+
type: "string",
|
|
65
|
+
},
|
|
66
|
+
{ name: "reserved", column: "reserved", start: 119, length: 131, type: "string" },
|
|
67
|
+
{ name: "postalCode", column: "postal_code", start: 250, length: 5, type: "string" },
|
|
68
|
+
{ name: "postOfficeName", column: "post_office_name", start: 255, length: 30, type: "string" },
|
|
69
|
+
{ name: "postOfficeNameSwe", column: "post_office_name_swe", start: 285, length: 30, type: "string" },
|
|
70
|
+
{ name: "postOfficeShortName", column: "post_office_short_name", start: 315, length: 12, type: "string" },
|
|
71
|
+
{ name: "postOfficeShortNameSwe", column: "post_office_short_name_swe", start: 327, length: 12, type: "string" },
|
|
72
|
+
{ name: "municipalityIdCode", column: "municipality_id_code", start: 339, length: 3, type: "string" },
|
|
73
|
+
{ name: "municipalityName", column: "municipality_name", start: 342, length: 20, type: "string" },
|
|
74
|
+
{ name: "municipalityNameSwe", column: "municipality_name_swe", start: 362, length: 20, type: "string" },
|
|
75
|
+
{ name: "regionId", column: "region_id", start: 382, length: 2, type: "string" },
|
|
76
|
+
{ name: "regionName", column: "region_name", start: 384, length: 30, type: "string" },
|
|
77
|
+
{ name: "regionNameSwe", column: "region_name_swe", start: 414, length: 30, type: "string" },
|
|
78
|
+
{ name: "changedAt", column: "changed_at", start: 444, length: 8, type: "date" },
|
|
79
|
+
{ name: "eventCode", column: "event_code", start: 452, length: 2, type: "integer" },
|
|
80
|
+
];
|
|
81
|
+
export const fileTypeToFields = {
|
|
82
|
+
BAF: addressFields,
|
|
83
|
+
PCF: zipcodeFields,
|
|
84
|
+
POM: zipcodeChangeFields,
|
|
85
|
+
};
|
|
86
|
+
export const fileTypeToTable = {
|
|
87
|
+
BAF: "addresses",
|
|
88
|
+
PCF: "postalcodes",
|
|
89
|
+
POM: "postalcode_changes",
|
|
90
|
+
};
|
|
91
|
+
//# sourceMappingURL=field-definitions.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"field-definitions.js","sourceRoot":"","sources":["../../src/parser/field-definitions.ts"],"names":[],"mappings":"AAUA,mDAAmD;AACnD,MAAM,CAAC,MAAM,aAAa,GAAsB;IAC9C,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE;IACzE,EAAE,IAAI,EAAE,WAAW,EAAE,MAAM,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE;IAC9E,EAAE,IAAI,EAAE,YAAY,EAAE,MAAM,EAAE,aAAa,EAAE,KAAK,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE;IACnF,EAAE,IAAI,EAAE,gBAAgB,EAAE,MAAM,EAAE,kBAAkB,EAAE,KAAK,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;IAC7F,EAAE,IAAI,EAAE,mBAAmB,EAAE,MAAM,EAAE,sBAAsB,EAAE,KAAK,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;IACpG,EAAE,IAAI,EAAE,qBAAqB,EAAE,MAAM,EAAE,wBAAwB,EAAE,KAAK,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;IACxG,EAAE,IAAI,EAAE,wBAAwB,EAAE,MAAM,EAAE,4BAA4B,EAAE,KAAK,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;IAC/G,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;IAC9E,EAAE,IAAI,EAAE,YAAY,EAAE,MAAM,EAAE,aAAa,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;IACrF,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;IAC5E,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;IAC5E,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE;IAC/E,EAAE,IAAI,EAAE,yBAAyB,EAAE,MAAM,EAAE,4BAA4B,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE;IACjH,EAAE,IAAI,EAAE,yBAAyB,EAAE,MAAM,EAAE,4BAA4B,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE;IAChH,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,EAAE,WAAW,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE;IAChF,EAAE,IAAI,EAAE,yBAAyB,EAAE,MAAM,EAAE,4BAA4B,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE;IACjH,EAAE,IAAI,EAAE,yBAAyB,EAAE,MAAM,EAAE,4BAA4B,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE;IAChH,EAAE,IAAI,EAAE,wBAAwB,EAAE,MAAM,EAAE,2BAA2B,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE;IAC/G,EAAE,IAAI,EAAE,wBAAwB,EAAE,MAAM,EAAE,2BAA2B,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE;IAC9G,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,EAAE,WAAW,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE;IAChF,EAAE,IAAI,EAAE,wBAAwB,EAAE,MAAM,EAAE,2BAA2B,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE;IAC/G,EAAE,IAAI,EAAE,wBAAwB,EAAE,MAAM,EAAE,2BAA2B,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE;IAC9G,EAAE,IAAI,EAAE,oBAAoB,EAAE,MAAM,EAAE,sBAAsB,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE;IACrG,EAAE,IAAI,EAAE,kBAAkB,EAAE,MAAM,EAAE,mBAAmB,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;IACjG,EAAE,IAAI,EAAE,qBAAqB,EAAE,MAAM,EAAE,uBAAuB,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;CACzG,CAAC;AAEF,uDAAuD;AACvD,MAAM,CAAC,MAAM,aAAa,GAAsB;IAC9C,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE;IACzE,EAAE,IAAI,EAAE,WAAW,EAAE,MAAM,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE;IAC9E,EAAE,IAAI,EAAE,YAAY,EAAE,MAAM,EAAE,aAAa,EAAE,KAAK,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE;IACnF,EAAE,IAAI,EAAE,gBAAgB,EAAE,MAAM,EAAE,kBAAkB,EAAE,KAAK,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;IAC7F,EAAE,IAAI,EAAE,mBAAmB,EAAE,MAAM,EAAE,sBAAsB,EAAE,KAAK,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;IACpG,EAAE,IAAI,EAAE,qBAAqB,EAAE,MAAM,EAAE,wBAAwB,EAAE,KAAK,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;IACxG,EAAE,IAAI,EAAE,wBAAwB,EAAE,MAAM,EAAE,4BAA4B,EAAE,KAAK,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;IAC/G,EAAE,IAAI,EAAE,kBAAkB,EAAE,MAAM,EAAE,qBAAqB,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE;IAChG,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,EAAE,WAAW,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE;IACjF,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,EAAE,WAAW,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE;IAChF,EAAE,IAAI,EAAE,YAAY,EAAE,MAAM,EAAE,aAAa,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;IACrF,EAAE,IAAI,EAAE,eAAe,EAAE,MAAM,EAAE,iBAAiB,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;IAC5F,EAAE,IAAI,EAAE,oBAAoB,EAAE,MAAM,EAAE,sBAAsB,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE;IACrG,EAAE,IAAI,EAAE,kBAAkB,EAAE,MAAM,EAAE,mBAAmB,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;IACjG,EAAE,IAAI,EAAE,qBAAqB,EAAE,MAAM,EAAE,uBAAuB,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;IACxG,EAAE,IAAI,EAAE,sBAAsB,EAAE,MAAM,EAAE,uBAAuB,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE;CAC1G,CAAC;AAEF,8DAA8D;AAC9D,MAAM,CAAC,MAAM,mBAAmB,GAAsB;IACpD,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE;IACzE,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE;IACvE,EAAE,IAAI,EAAE,WAAW,EAAE,MAAM,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE;IAC9E,EAAE,IAAI,EAAE,mBAAmB,EAAE,MAAM,EAAE,qBAAqB,EAAE,KAAK,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE;IAChG,EAAE,IAAI,EAAE,iBAAiB,EAAE,MAAM,EAAE,mBAAmB,EAAE,KAAK,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE;IAC5F,EAAE,IAAI,EAAE,eAAe,EAAE,MAAM,EAAE,iBAAiB,EAAE,KAAK,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE;IAC1F,EAAE,IAAI,EAAE,mBAAmB,EAAE,MAAM,EAAE,sBAAsB,EAAE,KAAK,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;IACpG,EAAE,IAAI,EAAE,sBAAsB,EAAE,MAAM,EAAE,0BAA0B,EAAE,KAAK,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;IAC3G,EAAE,IAAI,EAAE,wBAAwB,EAAE,MAAM,EAAE,4BAA4B,EAAE,KAAK,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;IAC/G;QACE,IAAI,EAAE,2BAA2B;QACjC,MAAM,EAAE,gCAAgC;QACxC,KAAK,EAAE,GAAG;QACV,MAAM,EAAE,EAAE;QACV,IAAI,EAAE,QAAQ;KACf;IACD,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE;IACjF,EAAE,IAAI,EAAE,YAAY,EAAE,MAAM,EAAE,aAAa,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE;IACpF,EAAE,IAAI,EAAE,gBAAgB,EAAE,MAAM,EAAE,kBAAkB,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;IAC9F,EAAE,IAAI,EAAE,mBAAmB,EAAE,MAAM,EAAE,sBAAsB,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;IACrG,EAAE,IAAI,EAAE,qBAAqB,EAAE,MAAM,EAAE,wBAAwB,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;IACzG,EAAE,IAAI,EAAE,wBAAwB,EAAE,MAAM,EAAE,4BAA4B,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;IAChH,EAAE,IAAI,EAAE,oBAAoB,EAAE,MAAM,EAAE,sBAAsB,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE;IACrG,EAAE,IAAI,EAAE,kBAAkB,EAAE,MAAM,EAAE,mBAAmB,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;IACjG,EAAE,IAAI,EAAE,qBAAqB,EAAE,MAAM,EAAE,uBAAuB,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;IACxG,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,EAAE,WAAW,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE;IAChF,EAAE,IAAI,EAAE,YAAY,EAAE,MAAM,EAAE,aAAa,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;IACrF,EAAE,IAAI,EAAE,eAAe,EAAE,MAAM,EAAE,iBAAiB,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE;IAC5F,EAAE,IAAI,EAAE,WAAW,EAAE,MAAM,EAAE,YAAY,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE;IAChF,EAAE,IAAI,EAAE,WAAW,EAAE,MAAM,EAAE,YAAY,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE;CACpF,CAAC;AAEF,MAAM,CAAC,MAAM,gBAAgB,GAAwC;IACnE,GAAG,EAAE,aAAa;IAClB,GAAG,EAAE,aAAa;IAClB,GAAG,EAAE,mBAAmB;CACzB,CAAC;AAEF,MAAM,CAAC,MAAM,eAAe,GAA6B;IACvD,GAAG,EAAE,WAAW;IAChB,GAAG,EAAE,aAAa;IAClB,GAAG,EAAE,oBAAoB;CAC1B,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"file-converter.d.ts","sourceRoot":"","sources":["../../src/parser/file-converter.ts"],"names":[],"mappings":"AAIA,wBAAsB,WAAW,CAAC,SAAS,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAOtF"}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import { createReadStream, createWriteStream } from "node:fs";
|
|
2
|
+
import { pipeline } from "node:stream/promises";
|
|
3
|
+
import iconv from "iconv-lite";
|
|
4
|
+
export async function convertFile(inputPath, outputPath) {
|
|
5
|
+
await pipeline(createReadStream(inputPath), iconv.decodeStream("latin1"), iconv.encodeStream("utf8"), createWriteStream(outputPath));
|
|
6
|
+
}
|
|
7
|
+
//# sourceMappingURL=file-converter.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"file-converter.js","sourceRoot":"","sources":["../../src/parser/file-converter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,SAAS,CAAC;AAC9D,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAChD,OAAO,KAAK,MAAM,YAAY,CAAC;AAE/B,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,SAAiB,EAAE,UAAkB;IACrE,MAAM,QAAQ,CACZ,gBAAgB,CAAC,SAAS,CAAC,EAC3B,KAAK,CAAC,YAAY,CAAC,QAAQ,CAAC,EAC5B,KAAK,CAAC,YAAY,CAAC,MAAM,CAAC,EAC1B,iBAAiB,CAAC,UAAU,CAAC,CAC9B,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import type { FieldDefinition } from "./field-definitions.js";
|
|
2
|
+
export declare function castValue(value: string, type: FieldDefinition["type"]): string | number | null;
|
|
3
|
+
export declare function parseLine(line: string, fields: FieldDefinition[]): Record<string, string | number | null>;
|
|
4
|
+
//# sourceMappingURL=line-parser.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"line-parser.d.ts","sourceRoot":"","sources":["../../src/parser/line-parser.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,wBAAwB,CAAC;AAE9D,wBAAgB,SAAS,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,eAAe,CAAC,MAAM,CAAC,GAAG,MAAM,GAAG,MAAM,GAAG,IAAI,CAiB9F;AAED,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,eAAe,EAAE,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,GAAG,IAAI,CAAC,CAWzG"}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
export function castValue(value, type) {
|
|
2
|
+
if (value === "") {
|
|
3
|
+
return null;
|
|
4
|
+
}
|
|
5
|
+
switch (type) {
|
|
6
|
+
case "integer": {
|
|
7
|
+
const num = Number.parseInt(value, 10);
|
|
8
|
+
return Number.isNaN(num) ? null : num;
|
|
9
|
+
}
|
|
10
|
+
case "date": {
|
|
11
|
+
const matches = value.match(/(\d{4})(\d{2})(\d{2})/);
|
|
12
|
+
return matches ? `${matches[1]}-${matches[2]}-${matches[3]}` : null;
|
|
13
|
+
}
|
|
14
|
+
default:
|
|
15
|
+
return value;
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
export function parseLine(line, fields) {
|
|
19
|
+
const row = {};
|
|
20
|
+
for (const field of fields) {
|
|
21
|
+
if (field.start !== undefined) {
|
|
22
|
+
const raw = line.substring(field.start - 1, field.start - 1 + field.length).trim();
|
|
23
|
+
row[field.name] = castValue(raw, field.type);
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
return row;
|
|
27
|
+
}
|
|
28
|
+
//# sourceMappingURL=line-parser.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"line-parser.js","sourceRoot":"","sources":["../../src/parser/line-parser.ts"],"names":[],"mappings":"AAEA,MAAM,UAAU,SAAS,CAAC,KAAa,EAAE,IAA6B;IACpE,IAAI,KAAK,KAAK,EAAE,EAAE,CAAC;QACjB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,QAAQ,IAAI,EAAE,CAAC;QACb,KAAK,SAAS,EAAE,CAAC;YACf,MAAM,GAAG,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;YACvC,OAAO,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC;QACxC,CAAC;QACD,KAAK,MAAM,EAAE,CAAC;YACZ,MAAM,OAAO,GAAG,KAAK,CAAC,KAAK,CAAC,uBAAuB,CAAC,CAAC;YACrD,OAAO,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,CAAC,CAAC,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;QACtE,CAAC;QACD;YACE,OAAO,KAAK,CAAC;IACjB,CAAC;AACH,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,IAAY,EAAE,MAAyB;IAC/D,MAAM,GAAG,GAA2C,EAAE,CAAC;IAEvD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,IAAI,KAAK,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAC9B,MAAM,GAAG,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,EAAE,KAAK,CAAC,KAAK,GAAG,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC;YACnF,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;QAC/C,CAAC;IACH,CAAC;IAED,OAAO,GAAG,CAAC;AACb,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"unzipper.d.ts","sourceRoot":"","sources":["../../src/parser/unzipper.ts"],"names":[],"mappings":"AAGA,wBAAsB,UAAU,CAAC,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAOlF"}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { createReadStream } from "node:fs";
|
|
2
|
+
import unzip from "unzipper";
|
|
3
|
+
export async function extractZip(zipPath, targetDir) {
|
|
4
|
+
return new Promise((resolve, reject) => {
|
|
5
|
+
createReadStream(zipPath)
|
|
6
|
+
.pipe(unzip.Extract({ path: targetDir }))
|
|
7
|
+
.on("close", resolve)
|
|
8
|
+
.on("error", reject);
|
|
9
|
+
});
|
|
10
|
+
}
|
|
11
|
+
//# sourceMappingURL=unzipper.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"unzipper.js","sourceRoot":"","sources":["../../src/parser/unzipper.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAC3C,OAAO,KAAK,MAAM,UAAU,CAAC;AAE7B,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,OAAe,EAAE,SAAiB;IACjE,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACrC,gBAAgB,CAAC,OAAO,CAAC;aACtB,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,CAAC;aACxC,EAAE,CAAC,OAAO,EAAE,OAAO,CAAC;aACpB,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IACzB,CAAC,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"chunks.d.ts","sourceRoot":"","sources":["../../src/utils/chunks.ts"],"names":[],"mappings":"AAAA,wBAAgB,eAAe,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE,IAAI,EAAE,MAAM,GAAG,CAAC,EAAE,EAAE,CAOlE"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
export function sliceIntoChunks(array, size) {
|
|
2
|
+
if (size <= 0)
|
|
3
|
+
throw new Error("Chunk size must be a positive number");
|
|
4
|
+
const result = [];
|
|
5
|
+
for (let i = 0; i < array.length; i += size) {
|
|
6
|
+
result.push(array.slice(i, i + size));
|
|
7
|
+
}
|
|
8
|
+
return result;
|
|
9
|
+
}
|
|
10
|
+
//# sourceMappingURL=chunks.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"chunks.js","sourceRoot":"","sources":["../../src/utils/chunks.ts"],"names":[],"mappings":"AAAA,MAAM,UAAU,eAAe,CAAI,KAAU,EAAE,IAAY;IACzD,IAAI,IAAI,IAAI,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;IACvE,MAAM,MAAM,GAAU,EAAE,CAAC;IACzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC;QAC5C,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC;IACxC,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"logger.d.ts","sourceRoot":"","sources":["../../src/utils/logger.ts"],"names":[],"mappings":"AAEA,eAAO,MAAM,MAAM;IACjB,IAAI,QAAQ,MAAM;IAClB,OAAO,QAAQ,MAAM;IACrB,IAAI,QAAQ,MAAM;IAClB,KAAK,QAAQ,MAAM;CACpB,CAAC"}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import pc from "picocolors";
|
|
2
|
+
export const logger = {
|
|
3
|
+
info: (msg) => console.info(pc.blue("info"), msg),
|
|
4
|
+
success: (msg) => console.info(pc.green("done"), msg),
|
|
5
|
+
warn: (msg) => console.warn(pc.yellow("warn"), msg),
|
|
6
|
+
error: (msg) => console.error(pc.red("error"), msg),
|
|
7
|
+
};
|
|
8
|
+
//# sourceMappingURL=logger.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"logger.js","sourceRoot":"","sources":["../../src/utils/logger.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,YAAY,CAAC;AAE5B,MAAM,CAAC,MAAM,MAAM,GAAG;IACpB,IAAI,EAAE,CAAC,GAAW,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,GAAG,CAAC;IACzD,OAAO,EAAE,CAAC,GAAW,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,GAAG,CAAC;IAC7D,IAAI,EAAE,CAAC,GAAW,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,GAAG,CAAC;IAC3D,KAAK,EAAE,CAAC,GAAW,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,GAAG,CAAC;CAC5D,CAAC"}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
export interface ProgressReporter {
|
|
2
|
+
startOverall(totalFiles: number): void;
|
|
3
|
+
startFile(filename: string): void;
|
|
4
|
+
updateFileStep(step: string): void;
|
|
5
|
+
updateFileProgress(step: string, current: number, total: number): void;
|
|
6
|
+
completeFile(): void;
|
|
7
|
+
stop(): void;
|
|
8
|
+
}
|
|
9
|
+
interface ProgressOptions {
|
|
10
|
+
silent?: boolean;
|
|
11
|
+
}
|
|
12
|
+
export declare function createProgress(options?: ProgressOptions): ProgressReporter;
|
|
13
|
+
export {};
|
|
14
|
+
//# sourceMappingURL=progress.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"progress.d.ts","sourceRoot":"","sources":["../../src/utils/progress.ts"],"names":[],"mappings":"AAGA,MAAM,WAAW,gBAAgB;IAC/B,YAAY,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IACvC,SAAS,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,kBAAkB,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACvE,YAAY,IAAI,IAAI,CAAC;IACrB,IAAI,IAAI,IAAI,CAAC;CACd;AAED,UAAU,eAAe;IACvB,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED,wBAAgB,cAAc,CAAC,OAAO,GAAE,eAAoB,GAAG,gBAAgB,CAK9E"}
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
import cliProgress from "cli-progress";
|
|
2
|
+
import pc from "picocolors";
|
|
3
|
+
export function createProgress(options = {}) {
|
|
4
|
+
if (options.silent) {
|
|
5
|
+
return createSilentProgress();
|
|
6
|
+
}
|
|
7
|
+
return createMultiBarProgress();
|
|
8
|
+
}
|
|
9
|
+
function createSilentProgress() {
|
|
10
|
+
return {
|
|
11
|
+
startOverall() { },
|
|
12
|
+
startFile() { },
|
|
13
|
+
updateFileStep() { },
|
|
14
|
+
updateFileProgress() { },
|
|
15
|
+
completeFile() { },
|
|
16
|
+
stop() { },
|
|
17
|
+
};
|
|
18
|
+
}
|
|
19
|
+
function createMultiBarProgress() {
|
|
20
|
+
const multi = new cliProgress.MultiBar({
|
|
21
|
+
clearOnComplete: false,
|
|
22
|
+
hideCursor: true,
|
|
23
|
+
format: "{label} {bar} {value}/{total} {suffix}",
|
|
24
|
+
barCompleteChar: "\u2588",
|
|
25
|
+
barIncompleteChar: "\u2591",
|
|
26
|
+
}, cliProgress.Presets.shades_grey);
|
|
27
|
+
let overallBar;
|
|
28
|
+
let fileBar;
|
|
29
|
+
let filesCompleted = 0;
|
|
30
|
+
return {
|
|
31
|
+
startOverall(totalFiles) {
|
|
32
|
+
filesCompleted = 0;
|
|
33
|
+
overallBar = multi.create(totalFiles, 0, {
|
|
34
|
+
label: pc.blue("files "),
|
|
35
|
+
suffix: "files",
|
|
36
|
+
});
|
|
37
|
+
},
|
|
38
|
+
startFile(filename) {
|
|
39
|
+
if (fileBar) {
|
|
40
|
+
multi.remove(fileBar);
|
|
41
|
+
}
|
|
42
|
+
fileBar = multi.create(100, 0, {
|
|
43
|
+
label: pc.cyan(filename.padEnd(7)),
|
|
44
|
+
suffix: "Starting...",
|
|
45
|
+
});
|
|
46
|
+
},
|
|
47
|
+
updateFileStep(step) {
|
|
48
|
+
fileBar?.update(0, { suffix: `${step}...`, total: 100 });
|
|
49
|
+
},
|
|
50
|
+
updateFileProgress(step, current, total) {
|
|
51
|
+
if (fileBar) {
|
|
52
|
+
fileBar.setTotal(total);
|
|
53
|
+
fileBar.update(current, { suffix: `${step} (${current}/${total})` });
|
|
54
|
+
}
|
|
55
|
+
},
|
|
56
|
+
completeFile() {
|
|
57
|
+
filesCompleted++;
|
|
58
|
+
if (fileBar) {
|
|
59
|
+
fileBar.update(fileBar.getTotal(), { suffix: pc.green("Done") });
|
|
60
|
+
}
|
|
61
|
+
overallBar?.update(filesCompleted);
|
|
62
|
+
},
|
|
63
|
+
stop() {
|
|
64
|
+
multi.stop();
|
|
65
|
+
},
|
|
66
|
+
};
|
|
67
|
+
}
|
|
68
|
+
//# sourceMappingURL=progress.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"progress.js","sourceRoot":"","sources":["../../src/utils/progress.ts"],"names":[],"mappings":"AAAA,OAAO,WAAW,MAAM,cAAc,CAAC;AACvC,OAAO,EAAE,MAAM,YAAY,CAAC;AAe5B,MAAM,UAAU,cAAc,CAAC,OAAO,GAAoB,EAAE;IAC1D,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QACnB,OAAO,oBAAoB,EAAE,CAAC;IAChC,CAAC;IACD,OAAO,sBAAsB,EAAE,CAAC;AAClC,CAAC;AAED,SAAS,oBAAoB;IAC3B,OAAO;QACL,YAAY,KAAI,CAAC;QACjB,SAAS,KAAI,CAAC;QACd,cAAc,KAAI,CAAC;QACnB,kBAAkB,KAAI,CAAC;QACvB,YAAY,KAAI,CAAC;QACjB,IAAI,KAAI,CAAC;KACV,CAAC;AACJ,CAAC;AAED,SAAS,sBAAsB;IAC7B,MAAM,KAAK,GAAG,IAAI,WAAW,CAAC,QAAQ,CACpC;QACE,eAAe,EAAE,KAAK;QACtB,UAAU,EAAE,IAAI;QAChB,MAAM,EAAE,wCAAwC;QAChD,eAAe,EAAE,QAAQ;QACzB,iBAAiB,EAAE,QAAQ;KAC5B,EACD,WAAW,CAAC,OAAO,CAAC,WAAW,CAChC,CAAC;IAEF,IAAI,UAA6C,CAAC;IAClD,IAAI,OAA0C,CAAC;IAC/C,IAAI,cAAc,GAAG,CAAC,CAAC;IAEvB,OAAO;QACL,YAAY,CAAC,UAAkB;YAC7B,cAAc,GAAG,CAAC,CAAC;YACnB,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC,EAAE;gBACvC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC;gBACzB,MAAM,EAAE,OAAO;aAChB,CAAC,CAAC;QACL,CAAC;QAED,SAAS,CAAC,QAAgB;YACxB,IAAI,OAAO,EAAE,CAAC;gBACZ,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YACxB,CAAC;YACD,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,EAAE;gBAC7B,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;gBAClC,MAAM,EAAE,aAAa;aACtB,CAAC,CAAC;QACL,CAAC;QAED,cAAc,CAAC,IAAY;YACzB,OAAO,EAAE,MAAM,CAAC,CAAC,EAAE,EAAE,MAAM,EAAE,GAAG,IAAI,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC;QAC3D,CAAC;QAED,kBAAkB,CAAC,IAAY,EAAE,OAAe,EAAE,KAAa;YAC7D,IAAI,OAAO,EAAE,CAAC;gBACZ,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;gBACxB,OAAO,CAAC,MAAM,CAAC,OAAO,EAAE,EAAE,MAAM,EAAE,GAAG,IAAI,KAAK,OAAO,IAAI,KAAK,GAAG,EAAE,CAAC,CAAC;YACvE,CAAC;QACH,CAAC;QAED,YAAY;YACV,cAAc,EAAE,CAAC;YACjB,IAAI,OAAO,EAAE,CAAC;gBACZ,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YACnE,CAAC;YACD,UAAU,EAAE,MAAM,CAAC,cAAc,CAAC,CAAC;QACrC,CAAC;QAED,IAAI;YACF,KAAK,CAAC,IAAI,EAAE,CAAC;QACf,CAAC;KACF,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"time.d.ts","sourceRoot":"","sources":["../../src/utils/time.ts"],"names":[],"mappings":"AAAA,wBAAgB,cAAc,CAAC,EAAE,EAAE,MAAM,GAAG,MAAM,CAYjD"}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
export function formatDuration(ms) {
|
|
2
|
+
const totalSeconds = ms / 1000;
|
|
3
|
+
const hours = Math.floor(totalSeconds / 3600);
|
|
4
|
+
const minutes = Math.floor((totalSeconds % 3600) / 60);
|
|
5
|
+
const seconds = totalSeconds % 60;
|
|
6
|
+
const parts = [];
|
|
7
|
+
if (hours > 0)
|
|
8
|
+
parts.push(`${hours}h`);
|
|
9
|
+
if (minutes > 0)
|
|
10
|
+
parts.push(`${minutes}m`);
|
|
11
|
+
parts.push(`${seconds.toFixed(3)}s`);
|
|
12
|
+
return parts.join(" ");
|
|
13
|
+
}
|
|
14
|
+
//# sourceMappingURL=time.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"time.js","sourceRoot":"","sources":["../../src/utils/time.ts"],"names":[],"mappings":"AAAA,MAAM,UAAU,cAAc,CAAC,EAAU;IACvC,MAAM,YAAY,GAAG,EAAE,GAAG,IAAI,CAAC;IAC/B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,GAAG,IAAI,CAAC,CAAC;IAC9C,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,YAAY,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;IACvD,MAAM,OAAO,GAAG,YAAY,GAAG,EAAE,CAAC;IAElC,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,KAAK,GAAG,CAAC;QAAE,KAAK,CAAC,IAAI,CAAC,GAAG,KAAK,GAAG,CAAC,CAAC;IACvC,IAAI,OAAO,GAAG,CAAC;QAAE,KAAK,CAAC,IAAI,CAAC,GAAG,OAAO,GAAG,CAAC,CAAC;IAC3C,KAAK,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IAErC,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AACzB,CAAC"}
|