posti 3.5.0 → 4.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (104) hide show
  1. package/{LICENSE.md → LICENSE} +1 -1
  2. package/README.md +333 -112
  3. package/dist/cli/args.d.ts +17 -0
  4. package/dist/cli/args.d.ts.map +1 -0
  5. package/dist/cli/args.js +80 -0
  6. package/dist/cli/args.js.map +1 -0
  7. package/dist/cli.d.ts +3 -0
  8. package/dist/cli.d.ts.map +1 -0
  9. package/dist/cli.js +85 -0
  10. package/dist/cli.js.map +1 -0
  11. package/dist/config/defaults.d.ts +2 -0
  12. package/dist/config/defaults.d.ts.map +1 -0
  13. package/dist/config/defaults.js +2 -0
  14. package/dist/config/defaults.js.map +1 -0
  15. package/dist/config/index.d.ts +5 -0
  16. package/dist/config/index.d.ts.map +1 -0
  17. package/dist/config/index.js +97 -1
  18. package/dist/config/index.js.map +1 -0
  19. package/dist/config/paths.d.ts +4 -0
  20. package/dist/config/paths.d.ts.map +1 -0
  21. package/dist/config/paths.js +16 -0
  22. package/dist/config/paths.js.map +1 -0
  23. package/dist/config/schema.d.ts +27 -0
  24. package/dist/config/schema.d.ts.map +1 -0
  25. package/dist/config/schema.js +24 -0
  26. package/dist/config/schema.js.map +1 -0
  27. package/dist/db/dialects.d.ts +15 -0
  28. package/dist/db/dialects.d.ts.map +1 -0
  29. package/dist/db/dialects.js +270 -0
  30. package/dist/db/dialects.js.map +1 -0
  31. package/dist/db/index.d.ts +27 -0
  32. package/dist/db/index.d.ts.map +1 -0
  33. package/dist/db/index.js +49 -0
  34. package/dist/db/index.js.map +1 -0
  35. package/dist/db/migrate.d.ts +8 -0
  36. package/dist/db/migrate.d.ts.map +1 -0
  37. package/dist/db/migrate.js +58 -0
  38. package/dist/db/migrate.js.map +1 -0
  39. package/dist/db/migrations/001_initial_schema.d.ts +3 -0
  40. package/dist/db/migrations/001_initial_schema.d.ts.map +1 -0
  41. package/dist/db/migrations/001_initial_schema.js +17 -0
  42. package/dist/db/migrations/001_initial_schema.js.map +1 -0
  43. package/dist/db/migrations/index.d.ts +3 -0
  44. package/dist/db/migrations/index.d.ts.map +1 -0
  45. package/dist/db/migrations/index.js +3 -0
  46. package/dist/db/migrations/index.js.map +1 -0
  47. package/dist/db/operations.d.ts +13 -0
  48. package/dist/db/operations.d.ts.map +1 -0
  49. package/dist/db/operations.js +179 -0
  50. package/dist/db/operations.js.map +1 -0
  51. package/dist/fetcher/url-parser.d.ts +9 -0
  52. package/dist/fetcher/url-parser.d.ts.map +1 -0
  53. package/dist/fetcher/url-parser.js +19 -0
  54. package/dist/fetcher/url-parser.js.map +1 -0
  55. package/dist/index.d.ts +15 -0
  56. package/dist/index.d.ts.map +1 -0
  57. package/dist/index.js +9 -1
  58. package/dist/index.js.map +1 -0
  59. package/dist/loader/index.d.ts +7 -0
  60. package/dist/loader/index.d.ts.map +1 -0
  61. package/dist/loader/index.js +145 -0
  62. package/dist/loader/index.js.map +1 -0
  63. package/dist/loader/state.d.ts +5 -0
  64. package/dist/loader/state.d.ts.map +1 -0
  65. package/dist/loader/state.js +26 -0
  66. package/dist/loader/state.js.map +1 -0
  67. package/dist/parser/field-definitions.d.ts +14 -0
  68. package/dist/parser/field-definitions.d.ts.map +1 -0
  69. package/dist/parser/field-definitions.js +91 -0
  70. package/dist/parser/field-definitions.js.map +1 -0
  71. package/dist/parser/file-converter.d.ts +2 -0
  72. package/dist/parser/file-converter.d.ts.map +1 -0
  73. package/dist/parser/file-converter.js +7 -0
  74. package/dist/parser/file-converter.js.map +1 -0
  75. package/dist/parser/line-parser.d.ts +4 -0
  76. package/dist/parser/line-parser.d.ts.map +1 -0
  77. package/dist/parser/line-parser.js +28 -0
  78. package/dist/parser/line-parser.js.map +1 -0
  79. package/dist/parser/unzipper.d.ts +2 -0
  80. package/dist/parser/unzipper.d.ts.map +1 -0
  81. package/dist/parser/unzipper.js +11 -0
  82. package/dist/parser/unzipper.js.map +1 -0
  83. package/dist/utils/chunks.d.ts +2 -0
  84. package/dist/utils/chunks.d.ts.map +1 -0
  85. package/dist/utils/chunks.js +10 -0
  86. package/dist/utils/chunks.js.map +1 -0
  87. package/dist/utils/logger.d.ts +7 -0
  88. package/dist/utils/logger.d.ts.map +1 -0
  89. package/dist/utils/logger.js +8 -0
  90. package/dist/utils/logger.js.map +1 -0
  91. package/dist/utils/progress.d.ts +14 -0
  92. package/dist/utils/progress.d.ts.map +1 -0
  93. package/dist/utils/progress.js +68 -0
  94. package/dist/utils/progress.js.map +1 -0
  95. package/dist/utils/time.d.ts +2 -0
  96. package/dist/utils/time.d.ts.map +1 -0
  97. package/dist/utils/time.js +14 -0
  98. package/dist/utils/time.js.map +1 -0
  99. package/package.json +76 -63
  100. package/CHANGELOG.md +0 -345
  101. package/dist/classes/Posti.js +0 -11
  102. package/dist/classes/Sequelize.js +0 -15
  103. package/dist/posti.js +0 -2
  104. package/dist/utils/index.js +0 -55
@@ -1,6 +1,6 @@
1
1
  MIT License
2
2
 
3
- Copyright (c) 2018 Kimmo Saari
3
+ Copyright (c) 2026 Kimmo Saari
4
4
 
5
5
  Permission is hereby granted, free of charge, to any person obtaining a copy
6
6
  of this software and associated documentation files (the "Software"), to deal
package/README.md CHANGED
@@ -1,160 +1,381 @@
1
- # POSTI
1
+ # posti
2
2
 
3
- ![pipeline status](https://posti.devaus.eu/pipeline.svg)
4
- [![coverage report](https://posti.devaus.eu/coverage.svg)](https://posti.devaus.eu/coverage)
5
- ![latest version](https://img.shields.io/github/package-json/v/kirbo/posti.svg)
6
- ![last commit](https://img.shields.io/github/last-commit/kirbo/posti.svg)
7
- ![total downloads](https://img.shields.io/npm/dt/posti.svg)
8
- ![dependencies](https://img.shields.io/librariesio/github/kirbo/posti.svg)
3
+ Finnish postal data loader — fetches Basic Address File (BAF), Postal Code File (PCF), and Postal Code Changes (POM) from [Posti's open data services](https://www.posti.fi/webpcode/) and loads them into your database.
9
4
 
10
- ## Description
5
+ Supports **PostgreSQL**, **MySQL**, **MariaDB**, and **SQLite**.
11
6
 
12
- `posti` is a tool to automagically download latest basic address data from [Finnish post - Posti](https://www.posti.fi/), to parse the files and insert the contents into database. **Check also [`posti-graphql`](https://www.npmjs.com/package/posti-graphql).**
7
+ ## Requirements
13
8
 
14
- ## Changelog
9
+ - Node.js >= 22.12 (the maintained LTS lines: 22, 24 and 26; CI tests all three)
10
+ - One of PostgreSQL, MySQL, MariaDB or SQLite (the database driver is an optional dependency)
15
11
 
16
- Check changes [here](./CHANGELOG.md).
12
+ ## Installation
17
13
 
18
- ## Posti
14
+ ```bash
15
+ # As a CLI tool
16
+ pnpm add -g posti
19
17
 
20
- **Postal Code Services - Homepage:**
18
+ # As a library
19
+ pnpm add posti
20
+ ```
21
21
 
22
- - [Finnish](https://support.posti.fi/fi/postinumeropalvelut/postinumerotiedostot.html)
23
- - [English](https://www.posti.fi/business/help-and-support/postal-code-services/postal-code-files.html)
24
- - [Swedish](https://www.posti.fi/foretag/hjalp-och-stod/postnummertjanster/postnummerfiler.html)
22
+ ## Quick Start
25
23
 
26
- **Postal Code Services – Service Description and Terms Of Use:**
24
+ ### CLI
27
25
 
28
- - [Finnish](https://www.posti.fi/liitteet-yrityksille/ehdot/postinumeropalvelut-palvelukuvaus-ja-kayttoehdot.pdf)
29
- - [English](https://www.posti.fi/liitteet-yrityksille/ehdot/postinumeropalvelut-palvelukuvaus-ja-kayttoehdot-en.pdf)
30
- - [Swedish](https://www.posti.fi/liitteet-yrityksille/ehdot/postinumeropalvelut-palvelukuvaus-ja-kayttoehdot-sv.pdf)
26
+ ```bash
27
+ # Load into PostgreSQL
28
+ posti --dialect postgres --host localhost --port 5432 --database posti --user posti --password posti
31
29
 
32
- **Postal Code Services – Frequently Asked Questions:**
30
+ # Load into MySQL
31
+ posti --dialect mysql --host localhost --port 3306 --database posti --user posti --password posti
33
32
 
34
- - [Finnish](https://www.posti.fi/liitteet-yrityksille/muut/postinumeropalvelut-faq.pdf)
35
- - [English](https://www.posti.fi/liitteet-yrityksille/muut/postinumeropalvelut-faq-en.pdf)
36
- - [Swedish](https://www.posti.fi/liitteet-yrityksille/muut/postinumeropalvelut-faq-sv.pdf)
33
+ # Load into SQLite (no server needed)
34
+ posti --dialect sqlite --database ./posti.db
37
35
 
38
- There are 3 different files:
36
+ # Force re-process even if files haven't changed
37
+ posti --force
39
38
 
40
- * **BAF** (Basic Address File) - Updated every saturday at 13:00 (1:00 PM) GMT+0
41
- * **PCF** (Postal Code File) - Updated every day at 13:00 (1:00 PM) GMT+0
42
- * **POM** (Postal Code Changes) - Updated on 2nd day of each month at 13:00 (1:00 PM) GMT+0
39
+ # Only load specific tables
40
+ posti --dialect postgres --database posti --tables postalcodes,addresses
41
+ ```
43
42
 
44
- This tool downloads the latest files and processes only the ones that have been updated,
45
- so you can setup the automation to be run as often as you like, but I would suggest you not to
46
- run this more than once per day.
43
+ ### CLI Options
47
44
 
48
- The files above can be found here:
45
+ ```
46
+ posti [command] [options]
47
+
48
+ Commands:
49
+ run Fetch and load postal data (default)
50
+ migrate Run database migrations only
51
+
52
+ Options:
53
+ --config <path> Path to config file
54
+ --force Force re-processing even if files haven't changed
55
+ --no-migrate Skip automatic database migrations on startup
56
+ --dialect <dialect> Database dialect (postgres, mysql, mariadb, sqlite)
57
+ --host <host> Database host
58
+ --port <port> Database port
59
+ --database <name> Database name (or file path for SQLite)
60
+ --user <user> Database user
61
+ --password <pass> Database password
62
+ --tables <list> Comma-separated tables (addresses,postalcodes,postalcode_changes)
63
+ --help Show this help
64
+ --version Show version
65
+ ```
49
66
 
50
- - https://www.posti.fi/webpcode/
67
+ ### Config File
68
+
69
+ Instead of CLI flags, you can use a config file. Create `posti.config.js`:
70
+
71
+ ```js
72
+ export default {
73
+ dialect: "postgres", // postgres, mysql, mariadb, sqlite
74
+ host: "localhost",
75
+ port: 5432,
76
+ user: "posti",
77
+ password: "posti",
78
+ database: "posti",
79
+ tablePrefix: "posti_",
80
+ migrate: true, // Auto-run migrations on startup (disable with false)
81
+
82
+ process: {
83
+ chunkSize: 1000, // Rows per bulk insert
84
+ deleteOnComplete: true, // Remove temp files after processing
85
+ },
86
+
87
+ // Which tables to process (default: all three)
88
+ tables: ["addresses", "postalcodes", "postalcode_changes"],
89
+ };
90
+ ```
51
91
 
52
- ## TIPS
92
+ Config file resolution order:
93
+ 1. `--config` CLI flag
94
+ 2. `POSTI_CONFIG` environment variable
95
+ 3. `$XDG_CONFIG_HOME/posti/config.js` (defaults to `~/.config/posti/config.js`)
96
+ 4. `./posti.config.js` in current directory
97
+
98
+ ### Environment Variables
99
+
100
+ All settings can be set via environment variables (overrides config file):
101
+
102
+ ```bash
103
+ POSTI_DIALECT=postgres
104
+ POSTI_HOST=localhost
105
+ POSTI_PORT=5432
106
+ POSTI_USER=posti
107
+ POSTI_PASSWORD=posti
108
+ POSTI_DATABASE=posti
109
+ POSTI_TABLE_PREFIX=posti_
110
+ POSTI_CONFIG=/path/to/config.js
111
+ ```
53
112
 
54
- While executing this tool, it will try to automatically find the config file from these paths, in this order:
55
- - `<path to your project>/posti.config.js`
56
- - `<path to your home dir>/.posti/config.js`
113
+ **Priority:** CLI flags > environment variables > config file > defaults
114
+
115
+ ### Progress
116
+
117
+ The CLI displays a multi-bar progress indicator during processing:
57
118
 
58
- You can also specify a custom path for the config file, with either:
59
- ```
60
- posti --config=/path/to/config
61
119
  ```
62
- ..or:
120
+ files ████████░░░░░░░░░░░░ 1/3 files
121
+ BAF_20250315.zip ████████████░░░░░░░░ Loading (4500/12000)
63
122
  ```
64
- config=/path/to/config posti
123
+
124
+ Steps shown per file: Downloading, Extracting, Converting encoding, Parsing, Loading (with row count), Upserting.
125
+
126
+ ### Programmatic Usage
127
+
128
+ ```typescript
129
+ import { run, loadConfig } from "posti";
130
+
131
+ const config = await loadConfig({
132
+ dialect: "postgres",
133
+ host: "localhost",
134
+ database: "posti",
135
+ user: "posti",
136
+ password: "posti",
137
+ });
138
+
139
+ // With progress bar (default)
140
+ await run(config, { force: true });
141
+
142
+ // Silent mode (no progress output, useful for scripts/programmatic use)
143
+ await run(config, { silent: true });
65
144
  ```
66
145
 
67
- You can also use `~` in the path, which will be converted as your home directory path automatically, so e.g.: `config=~/.posti.js` is going to be converted into e.g.: `config=/home/user/.posti.js`.
146
+ ## Database Migrations
68
147
 
69
- This tool uses [`Sequelize`](https://github.com/sequelize/sequelize), therefore this tool supports/should support:
70
- - MySQL / MariaDB
71
- - Postgres
72
- - SQLite
73
- - Microsoft SQL Server
148
+ Posti uses a lightweight custom migration system. Migration files are TypeScript modules in `src/db/migrations/` that use dialect-aware helpers to generate correct SQL for all supported databases.
74
149
 
75
- I've only tested the usage with MariaDB, so please, if you try any other of the supported databases, let me know if you're having issues and/or if its working, so that I can update this readme.
150
+ ### Automatic migrations
76
151
 
77
- ## Force update
152
+ By default, `posti run` applies any pending migrations before loading data. To disable this:
78
153
 
79
- You can bypass the checks and force this tool to process each of the files, by `force=true posti`.
154
+ ```bash
155
+ # Skip migrations on startup
156
+ posti run --no-migrate
80
157
 
81
- ## Installation as a dependency for your project
82
- 1. Install the dependency
83
- ```
84
- yarn add posti
85
- ```
86
- or:
87
- ```
88
- npm install posti
89
- ```
158
+ # Or in config file
159
+ export default {
160
+ migrate: false,
161
+ // ...
162
+ };
163
+ ```
90
164
 
91
- 2. Create a `posti.config.js` file in your project root with the contents of [posti.config.example.js](./posti.config.example.js)
165
+ ### Manual migrations
92
166
 
93
- 3. Create a `npm` script in your `package.json` file:
94
- ```javascript
95
- "scripts": {
96
- "update-posti": "posti"
97
- }
98
- ```
167
+ Run migrations explicitly without loading data:
99
168
 
100
- 4. Run the script:
101
- ```
102
- yarn update-posti
103
- ```
104
- or:
105
- ```
106
- npm run update-posti
107
- ```
169
+ ```bash
170
+ posti migrate --dialect postgres --host localhost --database posti --user posti --password posti
171
+ ```
108
172
 
109
- ## Installation as a global command
110
- 1. Install the package
173
+ ### How migrations work
174
+
175
+ - Migrations are tracked in a `{tablePrefix}schema_migrations` table (created automatically)
176
+ - Each migration runs once — already-applied migrations are skipped
177
+ - Migrations must be **idempotent** (use `IF NOT EXISTS` / `IF EXISTS`) since MySQL/MariaDB auto-commit DDL
178
+ - No down migrations — write a new forward migration to undo changes
179
+
180
+ ### Adding a new migration (development)
181
+
182
+ 1. Create `src/db/migrations/NNN_description.ts`:
183
+ ```typescript
184
+ import type { Migration } from "../migrate.js";
185
+ import { execSQL, quoteIdentifier } from "../operations.js";
186
+
187
+ export const myMigration: Migration = {
188
+ name: "002_add_index",
189
+ up: async (db, config) => {
190
+ const table = quoteIdentifier(db.dialect, `${config.tablePrefix}addresses`);
191
+ const col1 = quoteIdentifier(db.dialect, "postal_code");
192
+ const col2 = quoteIdentifier(db.dialect, "municipality_id_code");
193
+ await execSQL(db, `CREATE INDEX IF NOT EXISTS idx_example ON ${table} (${col1}, ${col2})`);
194
+ },
195
+ };
111
196
  ```
112
- yarn global add posti
113
- ```
114
- or:
115
- ```
116
- npm install -g posti
197
+ 2. Register it in `src/db/migrations/index.ts`:
198
+ ```typescript
199
+ import { myMigration } from "./002_add_index.js";
200
+ export const migrations = [initialSchema, myMigration];
117
201
  ```
202
+ 3. Add tests in `test/unit/migrate.test.ts`
118
203
 
119
- 2. Create a `.posti/config.js` file in your home dir with the contents of [posti.config.example.js](./posti.config.example.js)
204
+ ## Posti Postal Code Services
120
205
 
121
- 3. Run the script:
122
- ```
123
- posti
124
- ```
206
+ ### Documentation
125
207
 
126
- ## Links
208
+ **Homepage:**
209
+ - [Finnish](https://www.posti.fi/yrityksille/asiakastuki/postinumerotiedostot)
210
+ - [English](https://www.posti.fi/en/for-businesses/customer-support/postal-code-services)
211
+ - [Swedish](https://www.posti.fi/sv/for-foretag/kundsupport/postnummerfiler)
127
212
 
128
- - [Test coverages](https://posti.devaus.eu/coverage/)
129
- - [Screencapture of running the script](https://posti.devaus.eu/screencapture.gif)
130
- - [Screenshot of executed script](https://posti.devaus.eu/screenshot.png)
131
- - [Screenshot of executed tests](https://posti.devaus.eu/tests.png)
213
+ **Service Description and Terms of Use:**
214
+ - [Finnish](https://www.posti.fi/mzj3zpe8qb7p/1eKbwM2WAEY5AuGi5TrSZ7/33cfc2c66d2649af885b36e3935556a1/posti-postinumeropalvelut-palvelukuvaus-ja-kayttoehdot-20150101.pdf)
215
+ - [English](https://www.posti.fi/mzj3zpe8qb7p/1eKbwM2WAEY5AuGi5TrSZ7/c76a865cf5feb2c527a114b8615e9580/posti-postal-code-services-service-description-and-terms-of-use-20150101.pdf)
216
+ - [Swedish](https://www.posti.fi/mzj3zpe8qb7p/1eKbwM2WAEY5AuGi5TrSZ7/d1dfc920ff1cd5e3ef17f8234bd3d7aa/posti-postnummertjanster-tjanstebeskrivning-och-anvandnings-villkor-20150101.pdf)
132
217
 
133
- ## Disclaimer
218
+ ### Data Files
134
219
 
135
- I am not in any way affiliated with nor do I represent anything from [Finnish post - Posti](https://www.posti.fi/).
220
+ The files can be downloaded from: https://www.posti.fi/webpcode/
136
221
 
222
+ Posti provides three data files:
137
223
 
138
- ## License
224
+ | File | Table | Description | Update Frequency |
225
+ |------|-------|-------------|------------------|
226
+ | BAF | `addresses` | Basic Address File — street addresses with postal codes | Weekly (Saturdays after 15:00 EET/EEST) |
227
+ | PCF | `postalcodes` | Postal Code File — all postal codes with metadata | Daily (except Sundays) |
228
+ | POM | `postalcode_changes` | Postal Code Changes — history of postal code changes | Monthly (3rd business day) |
229
+
230
+ ## File Paths (XDG)
231
+
232
+ The tool follows the [XDG Base Directory Specification](https://specifications.freedesktop.org/basedir-spec/latest/):
233
+
234
+ | Purpose | Default Path |
235
+ |---------|-------------|
236
+ | Config | `~/.config/posti/config.js` |
237
+ | Cache (downloaded files) | `~/.cache/posti/` |
238
+ | State (processing history) | `~/.local/state/posti/latest.json` |
139
239
 
140
- MIT License
240
+ Override with `XDG_CONFIG_HOME`, `XDG_CACHE_HOME`, and `XDG_STATE_HOME` environment variables.
141
241
 
142
- Copyright (c) 2018 Kimmo Saari
242
+ ## Development
143
243
 
144
- Permission is hereby granted, free of charge, to any person obtaining a copy
145
- of this software and associated documentation files (the "Software"), to deal
146
- in the Software without restriction, including without limitation the rights
147
- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
148
- copies of the Software, and to permit persons to whom the Software is
149
- furnished to do so, subject to the following conditions:
244
+ ### Setup
150
245
 
151
- The above copyright notice and this permission notice shall be included in all
152
- copies or substantial portions of the Software.
246
+ The toolchain comes from [mise](https://mise.jdx.dev): [`mise.toml`](mise.toml) pins every
247
+ tool to an exact version (Node.js 26.13.0, the current LTS; pnpm 12; git-cliff; jq;
248
+ shellcheck) and the committed `mise.lock` records each download's checksum.
249
+
250
+ ```bash
251
+ git clone https://github.com/Kirbo/posti.git
252
+ cd posti
253
+ mise install
254
+ pnpm install
255
+ ```
256
+
257
+ Bump a tool: change its pin in `mise.toml`, run `mise lock` and commit both files (for pnpm,
258
+ also package.json `packageManager`; `mise run check-versions` says what disagrees). npm
259
+ packages and tools younger than a day are never installed (`pnpm-workspace.yaml`
260
+ `minimumReleaseAge`, `mise.toml` `minimum_release_age`).
261
+
262
+ ### Running Tests
263
+
264
+ ```bash
265
+ # All tests
266
+ pnpm test
267
+
268
+ # Unit tests only
269
+ pnpm test:unit
270
+
271
+ # Integration tests (all four databases; see Local Databases below)
272
+ pnpm test:integration
273
+
274
+ # SQLite integration tests only (no server needed)
275
+ pnpm test:sqlite
276
+
277
+ # Watch mode
278
+ pnpm test:watch
279
+
280
+ # With coverage
281
+ pnpm test:coverage
282
+ ```
283
+
284
+ ### Local Databases (Docker)
285
+
286
+ To run integration tests against MySQL, MariaDB, and PostgreSQL:
287
+
288
+ ```bash
289
+ # Start database containers
290
+ docker compose up -d
291
+
292
+ # Services:
293
+ # MySQL — localhost:3306 (user: posti, password: posti, db: posti)
294
+ # MariaDB — localhost:3307 (user: posti, password: posti, db: posti)
295
+ # PostgreSQL — localhost:5432 (user: posti, password: posti, db: posti)
296
+
297
+ # Run integration tests
298
+ pnpm test:integration
299
+
300
+ # Stop containers
301
+ docker compose down
302
+ ```
303
+
304
+ ### Other Commands
305
+
306
+ ```bash
307
+ # Type check
308
+ pnpm typecheck
309
+
310
+ # Lint (Biome; shellcheck for scripts/*.sh)
311
+ pnpm lint
312
+ pnpm lint:shell
313
+
314
+ # Build (compile TypeScript)
315
+ pnpm build
316
+
317
+ # Check the package as npm consumers get it (tarball contents, publint, attw, smoke test)
318
+ pnpm check:package
319
+
320
+ # Everything CI checks except the MySQL/MariaDB/PostgreSQL integration tests
321
+ mise run check
322
+
323
+ # Run CLI in dev mode
324
+ pnpm start -- --dialect sqlite --database ./test.db --force
325
+
326
+ # Run CLI in watch mode
327
+ pnpm dev -- --dialect sqlite --database ./test.db
328
+ ```
329
+
330
+ ### Project Structure
331
+
332
+ ```
333
+ src/
334
+ cli.ts # CLI entry point
335
+ cli/args.ts # Argument parser
336
+ index.ts # Library exports
337
+ config/ # Configuration loading, Zod schema, XDG paths
338
+ db/ # Database connection, operations, dialect-specific SQL, migrations
339
+ fetcher/ # URL parsing from Posti index page
340
+ parser/ # Fixed-width file parsing, encoding conversion, ZIP extraction
341
+ loader/ # Main orchestration pipeline, state tracking
342
+ utils/ # Logger, array chunking, time formatting
343
+
344
+ test/
345
+ unit/ # Unit tests (no external dependencies)
346
+ integration/ # Database integration tests
347
+
348
+ scripts/ # CI + release helpers (versions check, package check, next version, release)
349
+ docs/SECRETS.md # npm trusted publishing (no secrets in this repo)
350
+ ```
351
+
352
+ ### Commits and releases
353
+
354
+ Commits follow [Conventional Commits](https://www.conventionalcommits.org): they decide the
355
+ next version. On `master`, CI computes it with [git-cliff](https://git-cliff.org)
356
+ ([`cliff.toml`](cliff.toml), [`scripts/next-version.sh`](scripts/next-version.sh)): a breaking
357
+ change (`feat!:` or a `BREAKING CHANGE:` footer) bumps the major, `feat` the minor, `fix` and
358
+ `perf` the patch; other types alone release nothing. `mise run next-version` previews it.
359
+
360
+ The `release` job then stamps the version into package.json (in CI only: the repository keeps
361
+ `0.0.0-development`), **stages** it on [npm](https://www.npmjs.com/package/posti) and creates
362
+ the [GitHub release](https://github.com/Kirbo/posti/releases) and tag with the notes
363
+ ([`scripts/release.ts`](scripts/release.ts)). A staged version is not installable yet: **the
364
+ maintainer approves it on npmjs.com** ("Staged Packages" tab, with 2FA). Staging uses
365
+ [npm trusted publishing](https://docs.npmjs.com/trusted-publishers) from GitHub Actions: no
366
+ npm token exists. Setup and the npmjs.com values: [`docs/SECRETS.md`](docs/SECRETS.md).
367
+ `VERSION=x.y.z RELEASE_DRY_RUN=1 node scripts/release.ts` (after `pnpm build`) runs the
368
+ release checks and `npm stage publish --dry-run` locally.
369
+
370
+ ### CI
371
+
372
+ [`.github/workflows/ci.yml`](.github/workflows/ci.yml), on GitHub-hosted runners: `check`
373
+ (the toolchain from `mise.lock` in locked mode, a frozen install, lint, typecheck, build,
374
+ package check), `audit`, `test` on Node 22, 24 and 26 (unit + SQLite) and `integration`
375
+ against MySQL, MariaDB and PostgreSQL service containers; on `master` also `version` and
376
+ `release`. Pull requests get the same checks; a weekly run re-checks everything, including
377
+ `pnpm audit`. Docs-only changes run nothing.
378
+
379
+ ## License
153
380
 
154
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
155
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
156
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
157
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
158
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
159
- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
160
- SOFTWARE.
381
+ MIT
@@ -0,0 +1,17 @@
1
+ export interface ParsedArgs {
2
+ command: "run" | "migrate";
3
+ help: boolean;
4
+ version: boolean;
5
+ force: boolean;
6
+ noMigrate: boolean;
7
+ config?: string;
8
+ dialect?: string;
9
+ host?: string;
10
+ port?: number;
11
+ database?: string;
12
+ user?: string;
13
+ password?: string;
14
+ tables?: string[];
15
+ }
16
+ export declare function parseArgs(argv: string[]): ParsedArgs;
17
+ //# sourceMappingURL=args.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"args.d.ts","sourceRoot":"","sources":["../../src/cli/args.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,UAAU;IACzB,OAAO,EAAE,KAAK,GAAG,SAAS,CAAC;IAC3B,IAAI,EAAE,OAAO,CAAC;IACd,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,EAAE,OAAO,CAAC;IACf,SAAS,EAAE,OAAO,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;CACnB;AAMD,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,UAAU,CAwCpD"}
@@ -0,0 +1,80 @@
1
+ const COMMANDS = new Set(["run", "migrate"]);
2
+ const BOOLEAN_FLAGS = new Set(["help", "version", "force", "no-migrate"]);
3
+ const VALUE_FLAGS = new Set(["config", "dialect", "host", "port", "database", "user", "password", "tables"]);
4
+ export function parseArgs(argv) {
5
+ const result = {
6
+ command: "run",
7
+ help: false,
8
+ version: false,
9
+ force: false,
10
+ noMigrate: false,
11
+ };
12
+ for (let i = 0; i < argv.length; i++) {
13
+ const arg = argv[i];
14
+ if (arg && !arg.startsWith("--") && COMMANDS.has(arg)) {
15
+ result.command = arg;
16
+ continue;
17
+ }
18
+ if (!arg?.startsWith("--"))
19
+ continue;
20
+ if (arg.includes("=")) {
21
+ const eqIndex = arg.indexOf("=");
22
+ const name = arg.slice(2, eqIndex);
23
+ const value = arg.slice(eqIndex + 1);
24
+ applyFlag(result, name, value);
25
+ continue;
26
+ }
27
+ const name = arg.slice(2);
28
+ if (BOOLEAN_FLAGS.has(name)) {
29
+ applyFlag(result, name);
30
+ continue;
31
+ }
32
+ const nextArg = argv[i + 1];
33
+ if (VALUE_FLAGS.has(name) && nextArg !== undefined) {
34
+ i++;
35
+ applyFlag(result, name, nextArg);
36
+ }
37
+ }
38
+ return result;
39
+ }
40
+ function applyFlag(result, name, value) {
41
+ switch (name) {
42
+ case "help":
43
+ result.help = true;
44
+ break;
45
+ case "version":
46
+ result.version = true;
47
+ break;
48
+ case "force":
49
+ result.force = true;
50
+ break;
51
+ case "no-migrate":
52
+ result.noMigrate = true;
53
+ break;
54
+ case "config":
55
+ result.config = value;
56
+ break;
57
+ case "dialect":
58
+ result.dialect = value;
59
+ break;
60
+ case "host":
61
+ result.host = value;
62
+ break;
63
+ case "port":
64
+ result.port = value ? Number.parseInt(value, 10) : undefined;
65
+ break;
66
+ case "database":
67
+ result.database = value;
68
+ break;
69
+ case "user":
70
+ result.user = value;
71
+ break;
72
+ case "password":
73
+ result.password = value;
74
+ break;
75
+ case "tables":
76
+ result.tables = value?.split(",");
77
+ break;
78
+ }
79
+ }
80
+ //# sourceMappingURL=args.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"args.js","sourceRoot":"","sources":["../../src/cli/args.ts"],"names":[],"mappings":"AAgBA,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC,CAAC;AAC7C,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,YAAY,CAAC,CAAC,CAAC;AAC1E,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,CAAC,QAAQ,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,UAAU,EAAE,QAAQ,CAAC,CAAC,CAAC;AAE7G,MAAM,UAAU,SAAS,CAAC,IAAc;IACtC,MAAM,MAAM,GAAe;QACzB,OAAO,EAAE,KAAK;QACd,IAAI,EAAE,KAAK;QACX,OAAO,EAAE,KAAK;QACd,KAAK,EAAE,KAAK;QACZ,SAAS,EAAE,KAAK;KACjB,CAAC;IAEF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACrC,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;QACpB,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YACtD,MAAM,CAAC,OAAO,GAAG,GAAwB,CAAC;YAC1C,SAAS;QACX,CAAC;QACD,IAAI,CAAC,GAAG,EAAE,UAAU,CAAC,IAAI,CAAC;YAAE,SAAS;QAErC,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YACtB,MAAM,OAAO,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;YACjC,MAAM,IAAI,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;YACnC,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC;YACrC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;YAC/B,SAAS;QACX,CAAC;QAED,MAAM,IAAI,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAE1B,IAAI,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YAC5B,SAAS,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;YACxB,SAAS;QACX,CAAC;QAED,MAAM,OAAO,GAAG,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC5B,IAAI,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;YACnD,CAAC,EAAE,CAAC;YACJ,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;QACnC,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,SAAS,CAAC,MAAkB,EAAE,IAAY,EAAE,KAAc;IACjE,QAAQ,IAAI,EAAE,CAAC;QACb,KAAK,MAAM;YACT,MAAM,CAAC,IAAI,GAAG,IAAI,CAAC;YACnB,MAAM;QACR,KAAK,SAAS;YACZ,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC;YACtB,MAAM;QACR,KAAK,OAAO;YACV,MAAM,CAAC,KAAK,GAAG,IAAI,CAAC;YACpB,MAAM;QACR,KAAK,YAAY;YACf,MAAM,CAAC,SAAS,GAAG,IAAI,CAAC;YACxB,MAAM;QACR,KAAK,QAAQ;YACX,MAAM,CAAC,MAAM,GAAG,KAAK,CAAC;YACtB,MAAM;QACR,KAAK,SAAS;YACZ,MAAM,CAAC,OAAO,GAAG,KAAK,CAAC;YACvB,MAAM;QACR,KAAK,MAAM;YACT,MAAM,CAAC,IAAI,GAAG,KAAK,CAAC;YACpB,MAAM;QACR,KAAK,MAAM;YACT,MAAM,CAAC,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;YAC7D,MAAM;QACR,KAAK,UAAU;YACb,MAAM,CAAC,QAAQ,GAAG,KAAK,CAAC;YACxB,MAAM;QACR,KAAK,MAAM;YACT,MAAM,CAAC,IAAI,GAAG,KAAK,CAAC;YACpB,MAAM;QACR,KAAK,UAAU;YACb,MAAM,CAAC,QAAQ,GAAG,KAAK,CAAC;YACxB,MAAM;QACR,KAAK,QAAQ;YACX,MAAM,CAAC,MAAM,GAAG,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC;YAClC,MAAM;IACV,CAAC;AACH,CAAC"}
package/dist/cli.d.ts ADDED
@@ -0,0 +1,3 @@
1
+ #!/usr/bin/env node
2
+ export {};
3
+ //# sourceMappingURL=cli.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":""}