posti 3.5.1 → 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/{LICENSE.md → LICENSE} +1 -1
- package/README.md +333 -112
- package/dist/cli/args.d.ts +17 -0
- package/dist/cli/args.d.ts.map +1 -0
- package/dist/cli/args.js +80 -0
- package/dist/cli/args.js.map +1 -0
- package/dist/cli.d.ts +3 -0
- package/dist/cli.d.ts.map +1 -0
- package/dist/cli.js +85 -0
- package/dist/cli.js.map +1 -0
- package/dist/config/defaults.d.ts +2 -0
- package/dist/config/defaults.d.ts.map +1 -0
- package/dist/config/defaults.js +2 -0
- package/dist/config/defaults.js.map +1 -0
- package/dist/config/index.d.ts +5 -0
- package/dist/config/index.d.ts.map +1 -0
- package/dist/config/index.js +97 -1
- package/dist/config/index.js.map +1 -0
- package/dist/config/paths.d.ts +4 -0
- package/dist/config/paths.d.ts.map +1 -0
- package/dist/config/paths.js +16 -0
- package/dist/config/paths.js.map +1 -0
- package/dist/config/schema.d.ts +27 -0
- package/dist/config/schema.d.ts.map +1 -0
- package/dist/config/schema.js +24 -0
- package/dist/config/schema.js.map +1 -0
- package/dist/db/dialects.d.ts +15 -0
- package/dist/db/dialects.d.ts.map +1 -0
- package/dist/db/dialects.js +270 -0
- package/dist/db/dialects.js.map +1 -0
- package/dist/db/index.d.ts +27 -0
- package/dist/db/index.d.ts.map +1 -0
- package/dist/db/index.js +49 -0
- package/dist/db/index.js.map +1 -0
- package/dist/db/migrate.d.ts +8 -0
- package/dist/db/migrate.d.ts.map +1 -0
- package/dist/db/migrate.js +58 -0
- package/dist/db/migrate.js.map +1 -0
- package/dist/db/migrations/001_initial_schema.d.ts +3 -0
- package/dist/db/migrations/001_initial_schema.d.ts.map +1 -0
- package/dist/db/migrations/001_initial_schema.js +17 -0
- package/dist/db/migrations/001_initial_schema.js.map +1 -0
- package/dist/db/migrations/index.d.ts +3 -0
- package/dist/db/migrations/index.d.ts.map +1 -0
- package/dist/db/migrations/index.js +3 -0
- package/dist/db/migrations/index.js.map +1 -0
- package/dist/db/operations.d.ts +13 -0
- package/dist/db/operations.d.ts.map +1 -0
- package/dist/db/operations.js +179 -0
- package/dist/db/operations.js.map +1 -0
- package/dist/fetcher/url-parser.d.ts +9 -0
- package/dist/fetcher/url-parser.d.ts.map +1 -0
- package/dist/fetcher/url-parser.js +19 -0
- package/dist/fetcher/url-parser.js.map +1 -0
- package/dist/index.d.ts +15 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +9 -1
- package/dist/index.js.map +1 -0
- package/dist/loader/index.d.ts +7 -0
- package/dist/loader/index.d.ts.map +1 -0
- package/dist/loader/index.js +145 -0
- package/dist/loader/index.js.map +1 -0
- package/dist/loader/state.d.ts +5 -0
- package/dist/loader/state.d.ts.map +1 -0
- package/dist/loader/state.js +26 -0
- package/dist/loader/state.js.map +1 -0
- package/dist/parser/field-definitions.d.ts +14 -0
- package/dist/parser/field-definitions.d.ts.map +1 -0
- package/dist/parser/field-definitions.js +91 -0
- package/dist/parser/field-definitions.js.map +1 -0
- package/dist/parser/file-converter.d.ts +2 -0
- package/dist/parser/file-converter.d.ts.map +1 -0
- package/dist/parser/file-converter.js +7 -0
- package/dist/parser/file-converter.js.map +1 -0
- package/dist/parser/line-parser.d.ts +4 -0
- package/dist/parser/line-parser.d.ts.map +1 -0
- package/dist/parser/line-parser.js +28 -0
- package/dist/parser/line-parser.js.map +1 -0
- package/dist/parser/unzipper.d.ts +2 -0
- package/dist/parser/unzipper.d.ts.map +1 -0
- package/dist/parser/unzipper.js +11 -0
- package/dist/parser/unzipper.js.map +1 -0
- package/dist/utils/chunks.d.ts +2 -0
- package/dist/utils/chunks.d.ts.map +1 -0
- package/dist/utils/chunks.js +10 -0
- package/dist/utils/chunks.js.map +1 -0
- package/dist/utils/logger.d.ts +7 -0
- package/dist/utils/logger.d.ts.map +1 -0
- package/dist/utils/logger.js +8 -0
- package/dist/utils/logger.js.map +1 -0
- package/dist/utils/progress.d.ts +14 -0
- package/dist/utils/progress.d.ts.map +1 -0
- package/dist/utils/progress.js +68 -0
- package/dist/utils/progress.js.map +1 -0
- package/dist/utils/time.d.ts +2 -0
- package/dist/utils/time.d.ts.map +1 -0
- package/dist/utils/time.js +14 -0
- package/dist/utils/time.js.map +1 -0
- package/package.json +76 -63
- package/CHANGELOG.md +0 -351
- package/dist/classes/Posti.js +0 -11
- package/dist/classes/Sequelize.js +0 -15
- package/dist/posti.js +0 -2
- package/dist/utils/index.js +0 -55
package/{LICENSE.md → LICENSE}
RENAMED
package/README.md
CHANGED
|
@@ -1,160 +1,381 @@
|
|
|
1
|
-
#
|
|
1
|
+
# posti
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
[](https://posti.devaus.eu/coverage)
|
|
5
|
-

|
|
6
|
-

|
|
7
|
-

|
|
8
|
-

|
|
3
|
+
Finnish postal data loader — fetches Basic Address File (BAF), Postal Code File (PCF), and Postal Code Changes (POM) from [Posti's open data services](https://www.posti.fi/webpcode/) and loads them into your database.
|
|
9
4
|
|
|
10
|
-
|
|
5
|
+
Supports **PostgreSQL**, **MySQL**, **MariaDB**, and **SQLite**.
|
|
11
6
|
|
|
12
|
-
|
|
7
|
+
## Requirements
|
|
13
8
|
|
|
14
|
-
|
|
9
|
+
- Node.js >= 22.12 (the maintained LTS lines: 22, 24 and 26; CI tests all three)
|
|
10
|
+
- One of PostgreSQL, MySQL, MariaDB or SQLite (the database driver is an optional dependency)
|
|
15
11
|
|
|
16
|
-
|
|
12
|
+
## Installation
|
|
17
13
|
|
|
18
|
-
|
|
14
|
+
```bash
|
|
15
|
+
# As a CLI tool
|
|
16
|
+
pnpm add -g posti
|
|
19
17
|
|
|
20
|
-
|
|
18
|
+
# As a library
|
|
19
|
+
pnpm add posti
|
|
20
|
+
```
|
|
21
21
|
|
|
22
|
-
|
|
23
|
-
- [English](https://www.posti.fi/business/help-and-support/postal-code-services/postal-code-files.html)
|
|
24
|
-
- [Swedish](https://www.posti.fi/foretag/hjalp-och-stod/postnummertjanster/postnummerfiler.html)
|
|
22
|
+
## Quick Start
|
|
25
23
|
|
|
26
|
-
|
|
24
|
+
### CLI
|
|
27
25
|
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
26
|
+
```bash
|
|
27
|
+
# Load into PostgreSQL
|
|
28
|
+
posti --dialect postgres --host localhost --port 5432 --database posti --user posti --password posti
|
|
31
29
|
|
|
32
|
-
|
|
30
|
+
# Load into MySQL
|
|
31
|
+
posti --dialect mysql --host localhost --port 3306 --database posti --user posti --password posti
|
|
33
32
|
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
- [Swedish](https://www.posti.fi/liitteet-yrityksille/muut/postinumeropalvelut-faq-sv.pdf)
|
|
33
|
+
# Load into SQLite (no server needed)
|
|
34
|
+
posti --dialect sqlite --database ./posti.db
|
|
37
35
|
|
|
38
|
-
|
|
36
|
+
# Force re-process even if files haven't changed
|
|
37
|
+
posti --force
|
|
39
38
|
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
39
|
+
# Only load specific tables
|
|
40
|
+
posti --dialect postgres --database posti --tables postalcodes,addresses
|
|
41
|
+
```
|
|
43
42
|
|
|
44
|
-
|
|
45
|
-
so you can setup the automation to be run as often as you like, but I would suggest you not to
|
|
46
|
-
run this more than once per day.
|
|
43
|
+
### CLI Options
|
|
47
44
|
|
|
48
|
-
|
|
45
|
+
```
|
|
46
|
+
posti [command] [options]
|
|
47
|
+
|
|
48
|
+
Commands:
|
|
49
|
+
run Fetch and load postal data (default)
|
|
50
|
+
migrate Run database migrations only
|
|
51
|
+
|
|
52
|
+
Options:
|
|
53
|
+
--config <path> Path to config file
|
|
54
|
+
--force Force re-processing even if files haven't changed
|
|
55
|
+
--no-migrate Skip automatic database migrations on startup
|
|
56
|
+
--dialect <dialect> Database dialect (postgres, mysql, mariadb, sqlite)
|
|
57
|
+
--host <host> Database host
|
|
58
|
+
--port <port> Database port
|
|
59
|
+
--database <name> Database name (or file path for SQLite)
|
|
60
|
+
--user <user> Database user
|
|
61
|
+
--password <pass> Database password
|
|
62
|
+
--tables <list> Comma-separated tables (addresses,postalcodes,postalcode_changes)
|
|
63
|
+
--help Show this help
|
|
64
|
+
--version Show version
|
|
65
|
+
```
|
|
49
66
|
|
|
50
|
-
|
|
67
|
+
### Config File
|
|
68
|
+
|
|
69
|
+
Instead of CLI flags, you can use a config file. Create `posti.config.js`:
|
|
70
|
+
|
|
71
|
+
```js
|
|
72
|
+
export default {
|
|
73
|
+
dialect: "postgres", // postgres, mysql, mariadb, sqlite
|
|
74
|
+
host: "localhost",
|
|
75
|
+
port: 5432,
|
|
76
|
+
user: "posti",
|
|
77
|
+
password: "posti",
|
|
78
|
+
database: "posti",
|
|
79
|
+
tablePrefix: "posti_",
|
|
80
|
+
migrate: true, // Auto-run migrations on startup (disable with false)
|
|
81
|
+
|
|
82
|
+
process: {
|
|
83
|
+
chunkSize: 1000, // Rows per bulk insert
|
|
84
|
+
deleteOnComplete: true, // Remove temp files after processing
|
|
85
|
+
},
|
|
86
|
+
|
|
87
|
+
// Which tables to process (default: all three)
|
|
88
|
+
tables: ["addresses", "postalcodes", "postalcode_changes"],
|
|
89
|
+
};
|
|
90
|
+
```
|
|
51
91
|
|
|
52
|
-
|
|
92
|
+
Config file resolution order:
|
|
93
|
+
1. `--config` CLI flag
|
|
94
|
+
2. `POSTI_CONFIG` environment variable
|
|
95
|
+
3. `$XDG_CONFIG_HOME/posti/config.js` (defaults to `~/.config/posti/config.js`)
|
|
96
|
+
4. `./posti.config.js` in current directory
|
|
97
|
+
|
|
98
|
+
### Environment Variables
|
|
99
|
+
|
|
100
|
+
All settings can be set via environment variables (overrides config file):
|
|
101
|
+
|
|
102
|
+
```bash
|
|
103
|
+
POSTI_DIALECT=postgres
|
|
104
|
+
POSTI_HOST=localhost
|
|
105
|
+
POSTI_PORT=5432
|
|
106
|
+
POSTI_USER=posti
|
|
107
|
+
POSTI_PASSWORD=posti
|
|
108
|
+
POSTI_DATABASE=posti
|
|
109
|
+
POSTI_TABLE_PREFIX=posti_
|
|
110
|
+
POSTI_CONFIG=/path/to/config.js
|
|
111
|
+
```
|
|
53
112
|
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
113
|
+
**Priority:** CLI flags > environment variables > config file > defaults
|
|
114
|
+
|
|
115
|
+
### Progress
|
|
116
|
+
|
|
117
|
+
The CLI displays a multi-bar progress indicator during processing:
|
|
57
118
|
|
|
58
|
-
You can also specify a custom path for the config file, with either:
|
|
59
|
-
```
|
|
60
|
-
posti --config=/path/to/config
|
|
61
119
|
```
|
|
62
|
-
|
|
120
|
+
files ████████░░░░░░░░░░░░ 1/3 files
|
|
121
|
+
BAF_20250315.zip ████████████░░░░░░░░ Loading (4500/12000)
|
|
63
122
|
```
|
|
64
|
-
|
|
123
|
+
|
|
124
|
+
Steps shown per file: Downloading, Extracting, Converting encoding, Parsing, Loading (with row count), Upserting.
|
|
125
|
+
|
|
126
|
+
### Programmatic Usage
|
|
127
|
+
|
|
128
|
+
```typescript
|
|
129
|
+
import { run, loadConfig } from "posti";
|
|
130
|
+
|
|
131
|
+
const config = await loadConfig({
|
|
132
|
+
dialect: "postgres",
|
|
133
|
+
host: "localhost",
|
|
134
|
+
database: "posti",
|
|
135
|
+
user: "posti",
|
|
136
|
+
password: "posti",
|
|
137
|
+
});
|
|
138
|
+
|
|
139
|
+
// With progress bar (default)
|
|
140
|
+
await run(config, { force: true });
|
|
141
|
+
|
|
142
|
+
// Silent mode (no progress output, useful for scripts/programmatic use)
|
|
143
|
+
await run(config, { silent: true });
|
|
65
144
|
```
|
|
66
145
|
|
|
67
|
-
|
|
146
|
+
## Database Migrations
|
|
68
147
|
|
|
69
|
-
|
|
70
|
-
- MySQL / MariaDB
|
|
71
|
-
- Postgres
|
|
72
|
-
- SQLite
|
|
73
|
-
- Microsoft SQL Server
|
|
148
|
+
Posti uses a lightweight custom migration system. Migration files are TypeScript modules in `src/db/migrations/` that use dialect-aware helpers to generate correct SQL for all supported databases.
|
|
74
149
|
|
|
75
|
-
|
|
150
|
+
### Automatic migrations
|
|
76
151
|
|
|
77
|
-
|
|
152
|
+
By default, `posti run` applies any pending migrations before loading data. To disable this:
|
|
78
153
|
|
|
79
|
-
|
|
154
|
+
```bash
|
|
155
|
+
# Skip migrations on startup
|
|
156
|
+
posti run --no-migrate
|
|
80
157
|
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
```
|
|
88
|
-
npm install posti
|
|
89
|
-
```
|
|
158
|
+
# Or in config file
|
|
159
|
+
export default {
|
|
160
|
+
migrate: false,
|
|
161
|
+
// ...
|
|
162
|
+
};
|
|
163
|
+
```
|
|
90
164
|
|
|
91
|
-
|
|
165
|
+
### Manual migrations
|
|
92
166
|
|
|
93
|
-
|
|
94
|
-
```javascript
|
|
95
|
-
"scripts": {
|
|
96
|
-
"update-posti": "posti"
|
|
97
|
-
}
|
|
98
|
-
```
|
|
167
|
+
Run migrations explicitly without loading data:
|
|
99
168
|
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
```
|
|
104
|
-
or:
|
|
105
|
-
```
|
|
106
|
-
npm run update-posti
|
|
107
|
-
```
|
|
169
|
+
```bash
|
|
170
|
+
posti migrate --dialect postgres --host localhost --database posti --user posti --password posti
|
|
171
|
+
```
|
|
108
172
|
|
|
109
|
-
|
|
110
|
-
|
|
173
|
+
### How migrations work
|
|
174
|
+
|
|
175
|
+
- Migrations are tracked in a `{tablePrefix}schema_migrations` table (created automatically)
|
|
176
|
+
- Each migration runs once — already-applied migrations are skipped
|
|
177
|
+
- Migrations must be **idempotent** (use `IF NOT EXISTS` / `IF EXISTS`) since MySQL/MariaDB auto-commit DDL
|
|
178
|
+
- No down migrations — write a new forward migration to undo changes
|
|
179
|
+
|
|
180
|
+
### Adding a new migration (development)
|
|
181
|
+
|
|
182
|
+
1. Create `src/db/migrations/NNN_description.ts`:
|
|
183
|
+
```typescript
|
|
184
|
+
import type { Migration } from "../migrate.js";
|
|
185
|
+
import { execSQL, quoteIdentifier } from "../operations.js";
|
|
186
|
+
|
|
187
|
+
export const myMigration: Migration = {
|
|
188
|
+
name: "002_add_index",
|
|
189
|
+
up: async (db, config) => {
|
|
190
|
+
const table = quoteIdentifier(db.dialect, `${config.tablePrefix}addresses`);
|
|
191
|
+
const col1 = quoteIdentifier(db.dialect, "postal_code");
|
|
192
|
+
const col2 = quoteIdentifier(db.dialect, "municipality_id_code");
|
|
193
|
+
await execSQL(db, `CREATE INDEX IF NOT EXISTS idx_example ON ${table} (${col1}, ${col2})`);
|
|
194
|
+
},
|
|
195
|
+
};
|
|
111
196
|
```
|
|
112
|
-
|
|
113
|
-
```
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
npm install -g posti
|
|
197
|
+
2. Register it in `src/db/migrations/index.ts`:
|
|
198
|
+
```typescript
|
|
199
|
+
import { myMigration } from "./002_add_index.js";
|
|
200
|
+
export const migrations = [initialSchema, myMigration];
|
|
117
201
|
```
|
|
202
|
+
3. Add tests in `test/unit/migrate.test.ts`
|
|
118
203
|
|
|
119
|
-
|
|
204
|
+
## Posti Postal Code Services
|
|
120
205
|
|
|
121
|
-
|
|
122
|
-
```
|
|
123
|
-
posti
|
|
124
|
-
```
|
|
206
|
+
### Documentation
|
|
125
207
|
|
|
126
|
-
|
|
208
|
+
**Homepage:**
|
|
209
|
+
- [Finnish](https://www.posti.fi/yrityksille/asiakastuki/postinumerotiedostot)
|
|
210
|
+
- [English](https://www.posti.fi/en/for-businesses/customer-support/postal-code-services)
|
|
211
|
+
- [Swedish](https://www.posti.fi/sv/for-foretag/kundsupport/postnummerfiler)
|
|
127
212
|
|
|
128
|
-
|
|
129
|
-
- [
|
|
130
|
-
- [
|
|
131
|
-
- [
|
|
213
|
+
**Service Description and Terms of Use:**
|
|
214
|
+
- [Finnish](https://www.posti.fi/mzj3zpe8qb7p/1eKbwM2WAEY5AuGi5TrSZ7/33cfc2c66d2649af885b36e3935556a1/posti-postinumeropalvelut-palvelukuvaus-ja-kayttoehdot-20150101.pdf)
|
|
215
|
+
- [English](https://www.posti.fi/mzj3zpe8qb7p/1eKbwM2WAEY5AuGi5TrSZ7/c76a865cf5feb2c527a114b8615e9580/posti-postal-code-services-service-description-and-terms-of-use-20150101.pdf)
|
|
216
|
+
- [Swedish](https://www.posti.fi/mzj3zpe8qb7p/1eKbwM2WAEY5AuGi5TrSZ7/d1dfc920ff1cd5e3ef17f8234bd3d7aa/posti-postnummertjanster-tjanstebeskrivning-och-anvandnings-villkor-20150101.pdf)
|
|
132
217
|
|
|
133
|
-
|
|
218
|
+
### Data Files
|
|
134
219
|
|
|
135
|
-
|
|
220
|
+
The files can be downloaded from: https://www.posti.fi/webpcode/
|
|
136
221
|
|
|
222
|
+
Posti provides three data files:
|
|
137
223
|
|
|
138
|
-
|
|
224
|
+
| File | Table | Description | Update Frequency |
|
|
225
|
+
|------|-------|-------------|------------------|
|
|
226
|
+
| BAF | `addresses` | Basic Address File — street addresses with postal codes | Weekly (Saturdays after 15:00 EET/EEST) |
|
|
227
|
+
| PCF | `postalcodes` | Postal Code File — all postal codes with metadata | Daily (except Sundays) |
|
|
228
|
+
| POM | `postalcode_changes` | Postal Code Changes — history of postal code changes | Monthly (3rd business day) |
|
|
229
|
+
|
|
230
|
+
## File Paths (XDG)
|
|
231
|
+
|
|
232
|
+
The tool follows the [XDG Base Directory Specification](https://specifications.freedesktop.org/basedir-spec/latest/):
|
|
233
|
+
|
|
234
|
+
| Purpose | Default Path |
|
|
235
|
+
|---------|-------------|
|
|
236
|
+
| Config | `~/.config/posti/config.js` |
|
|
237
|
+
| Cache (downloaded files) | `~/.cache/posti/` |
|
|
238
|
+
| State (processing history) | `~/.local/state/posti/latest.json` |
|
|
139
239
|
|
|
140
|
-
|
|
240
|
+
Override with `XDG_CONFIG_HOME`, `XDG_CACHE_HOME`, and `XDG_STATE_HOME` environment variables.
|
|
141
241
|
|
|
142
|
-
|
|
242
|
+
## Development
|
|
143
243
|
|
|
144
|
-
|
|
145
|
-
of this software and associated documentation files (the "Software"), to deal
|
|
146
|
-
in the Software without restriction, including without limitation the rights
|
|
147
|
-
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
148
|
-
copies of the Software, and to permit persons to whom the Software is
|
|
149
|
-
furnished to do so, subject to the following conditions:
|
|
244
|
+
### Setup
|
|
150
245
|
|
|
151
|
-
The
|
|
152
|
-
|
|
246
|
+
The toolchain comes from [mise](https://mise.jdx.dev): [`mise.toml`](mise.toml) pins every
|
|
247
|
+
tool to an exact version (Node.js 26.13.0, the current LTS; pnpm 12; git-cliff; jq;
|
|
248
|
+
shellcheck) and the committed `mise.lock` records each download's checksum.
|
|
249
|
+
|
|
250
|
+
```bash
|
|
251
|
+
git clone https://github.com/Kirbo/posti.git
|
|
252
|
+
cd posti
|
|
253
|
+
mise install
|
|
254
|
+
pnpm install
|
|
255
|
+
```
|
|
256
|
+
|
|
257
|
+
Bump a tool: change its pin in `mise.toml`, run `mise lock` and commit both files (for pnpm,
|
|
258
|
+
also package.json `packageManager`; `mise run check-versions` says what disagrees). npm
|
|
259
|
+
packages and tools younger than a day are never installed (`pnpm-workspace.yaml`
|
|
260
|
+
`minimumReleaseAge`, `mise.toml` `minimum_release_age`).
|
|
261
|
+
|
|
262
|
+
### Running Tests
|
|
263
|
+
|
|
264
|
+
```bash
|
|
265
|
+
# All tests
|
|
266
|
+
pnpm test
|
|
267
|
+
|
|
268
|
+
# Unit tests only
|
|
269
|
+
pnpm test:unit
|
|
270
|
+
|
|
271
|
+
# Integration tests (all four databases; see Local Databases below)
|
|
272
|
+
pnpm test:integration
|
|
273
|
+
|
|
274
|
+
# SQLite integration tests only (no server needed)
|
|
275
|
+
pnpm test:sqlite
|
|
276
|
+
|
|
277
|
+
# Watch mode
|
|
278
|
+
pnpm test:watch
|
|
279
|
+
|
|
280
|
+
# With coverage
|
|
281
|
+
pnpm test:coverage
|
|
282
|
+
```
|
|
283
|
+
|
|
284
|
+
### Local Databases (Docker)
|
|
285
|
+
|
|
286
|
+
To run integration tests against MySQL, MariaDB, and PostgreSQL:
|
|
287
|
+
|
|
288
|
+
```bash
|
|
289
|
+
# Start database containers
|
|
290
|
+
docker compose up -d
|
|
291
|
+
|
|
292
|
+
# Services:
|
|
293
|
+
# MySQL — localhost:3306 (user: posti, password: posti, db: posti)
|
|
294
|
+
# MariaDB — localhost:3307 (user: posti, password: posti, db: posti)
|
|
295
|
+
# PostgreSQL — localhost:5432 (user: posti, password: posti, db: posti)
|
|
296
|
+
|
|
297
|
+
# Run integration tests
|
|
298
|
+
pnpm test:integration
|
|
299
|
+
|
|
300
|
+
# Stop containers
|
|
301
|
+
docker compose down
|
|
302
|
+
```
|
|
303
|
+
|
|
304
|
+
### Other Commands
|
|
305
|
+
|
|
306
|
+
```bash
|
|
307
|
+
# Type check
|
|
308
|
+
pnpm typecheck
|
|
309
|
+
|
|
310
|
+
# Lint (Biome; shellcheck for scripts/*.sh)
|
|
311
|
+
pnpm lint
|
|
312
|
+
pnpm lint:shell
|
|
313
|
+
|
|
314
|
+
# Build (compile TypeScript)
|
|
315
|
+
pnpm build
|
|
316
|
+
|
|
317
|
+
# Check the package as npm consumers get it (tarball contents, publint, attw, smoke test)
|
|
318
|
+
pnpm check:package
|
|
319
|
+
|
|
320
|
+
# Everything CI checks except the MySQL/MariaDB/PostgreSQL integration tests
|
|
321
|
+
mise run check
|
|
322
|
+
|
|
323
|
+
# Run CLI in dev mode
|
|
324
|
+
pnpm start -- --dialect sqlite --database ./test.db --force
|
|
325
|
+
|
|
326
|
+
# Run CLI in watch mode
|
|
327
|
+
pnpm dev -- --dialect sqlite --database ./test.db
|
|
328
|
+
```
|
|
329
|
+
|
|
330
|
+
### Project Structure
|
|
331
|
+
|
|
332
|
+
```
|
|
333
|
+
src/
|
|
334
|
+
cli.ts # CLI entry point
|
|
335
|
+
cli/args.ts # Argument parser
|
|
336
|
+
index.ts # Library exports
|
|
337
|
+
config/ # Configuration loading, Zod schema, XDG paths
|
|
338
|
+
db/ # Database connection, operations, dialect-specific SQL, migrations
|
|
339
|
+
fetcher/ # URL parsing from Posti index page
|
|
340
|
+
parser/ # Fixed-width file parsing, encoding conversion, ZIP extraction
|
|
341
|
+
loader/ # Main orchestration pipeline, state tracking
|
|
342
|
+
utils/ # Logger, array chunking, time formatting
|
|
343
|
+
|
|
344
|
+
test/
|
|
345
|
+
unit/ # Unit tests (no external dependencies)
|
|
346
|
+
integration/ # Database integration tests
|
|
347
|
+
|
|
348
|
+
scripts/ # CI + release helpers (versions check, package check, next version, release)
|
|
349
|
+
docs/SECRETS.md # npm trusted publishing (no secrets in this repo)
|
|
350
|
+
```
|
|
351
|
+
|
|
352
|
+
### Commits and releases
|
|
353
|
+
|
|
354
|
+
Commits follow [Conventional Commits](https://www.conventionalcommits.org): they decide the
|
|
355
|
+
next version. On `master`, CI computes it with [git-cliff](https://git-cliff.org)
|
|
356
|
+
([`cliff.toml`](cliff.toml), [`scripts/next-version.sh`](scripts/next-version.sh)): a breaking
|
|
357
|
+
change (`feat!:` or a `BREAKING CHANGE:` footer) bumps the major, `feat` the minor, `fix` and
|
|
358
|
+
`perf` the patch; other types alone release nothing. `mise run next-version` previews it.
|
|
359
|
+
|
|
360
|
+
The `release` job then stamps the version into package.json (in CI only: the repository keeps
|
|
361
|
+
`0.0.0-development`), **stages** it on [npm](https://www.npmjs.com/package/posti) and creates
|
|
362
|
+
the [GitHub release](https://github.com/Kirbo/posti/releases) and tag with the notes
|
|
363
|
+
([`scripts/release.ts`](scripts/release.ts)). A staged version is not installable yet: **the
|
|
364
|
+
maintainer approves it on npmjs.com** ("Staged Packages" tab, with 2FA). Staging uses
|
|
365
|
+
[npm trusted publishing](https://docs.npmjs.com/trusted-publishers) from GitHub Actions: no
|
|
366
|
+
npm token exists. Setup and the npmjs.com values: [`docs/SECRETS.md`](docs/SECRETS.md).
|
|
367
|
+
`VERSION=x.y.z RELEASE_DRY_RUN=1 node scripts/release.ts` (after `pnpm build`) runs the
|
|
368
|
+
release checks and `npm stage publish --dry-run` locally.
|
|
369
|
+
|
|
370
|
+
### CI
|
|
371
|
+
|
|
372
|
+
[`.github/workflows/ci.yml`](.github/workflows/ci.yml), on GitHub-hosted runners: `check`
|
|
373
|
+
(the toolchain from `mise.lock` in locked mode, a frozen install, lint, typecheck, build,
|
|
374
|
+
package check), `audit`, `test` on Node 22, 24 and 26 (unit + SQLite) and `integration`
|
|
375
|
+
against MySQL, MariaDB and PostgreSQL service containers; on `master` also `version` and
|
|
376
|
+
`release`. Pull requests get the same checks; a weekly run re-checks everything, including
|
|
377
|
+
`pnpm audit`. Docs-only changes run nothing.
|
|
378
|
+
|
|
379
|
+
## License
|
|
153
380
|
|
|
154
|
-
|
|
155
|
-
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
156
|
-
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
157
|
-
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
158
|
-
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
159
|
-
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
160
|
-
SOFTWARE.
|
|
381
|
+
MIT
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
export interface ParsedArgs {
|
|
2
|
+
command: "run" | "migrate";
|
|
3
|
+
help: boolean;
|
|
4
|
+
version: boolean;
|
|
5
|
+
force: boolean;
|
|
6
|
+
noMigrate: boolean;
|
|
7
|
+
config?: string;
|
|
8
|
+
dialect?: string;
|
|
9
|
+
host?: string;
|
|
10
|
+
port?: number;
|
|
11
|
+
database?: string;
|
|
12
|
+
user?: string;
|
|
13
|
+
password?: string;
|
|
14
|
+
tables?: string[];
|
|
15
|
+
}
|
|
16
|
+
export declare function parseArgs(argv: string[]): ParsedArgs;
|
|
17
|
+
//# sourceMappingURL=args.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"args.d.ts","sourceRoot":"","sources":["../../src/cli/args.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,UAAU;IACzB,OAAO,EAAE,KAAK,GAAG,SAAS,CAAC;IAC3B,IAAI,EAAE,OAAO,CAAC;IACd,OAAO,EAAE,OAAO,CAAC;IACjB,KAAK,EAAE,OAAO,CAAC;IACf,SAAS,EAAE,OAAO,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;CACnB;AAMD,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,UAAU,CAwCpD"}
|
package/dist/cli/args.js
ADDED
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
const COMMANDS = new Set(["run", "migrate"]);
|
|
2
|
+
const BOOLEAN_FLAGS = new Set(["help", "version", "force", "no-migrate"]);
|
|
3
|
+
const VALUE_FLAGS = new Set(["config", "dialect", "host", "port", "database", "user", "password", "tables"]);
|
|
4
|
+
export function parseArgs(argv) {
|
|
5
|
+
const result = {
|
|
6
|
+
command: "run",
|
|
7
|
+
help: false,
|
|
8
|
+
version: false,
|
|
9
|
+
force: false,
|
|
10
|
+
noMigrate: false,
|
|
11
|
+
};
|
|
12
|
+
for (let i = 0; i < argv.length; i++) {
|
|
13
|
+
const arg = argv[i];
|
|
14
|
+
if (arg && !arg.startsWith("--") && COMMANDS.has(arg)) {
|
|
15
|
+
result.command = arg;
|
|
16
|
+
continue;
|
|
17
|
+
}
|
|
18
|
+
if (!arg?.startsWith("--"))
|
|
19
|
+
continue;
|
|
20
|
+
if (arg.includes("=")) {
|
|
21
|
+
const eqIndex = arg.indexOf("=");
|
|
22
|
+
const name = arg.slice(2, eqIndex);
|
|
23
|
+
const value = arg.slice(eqIndex + 1);
|
|
24
|
+
applyFlag(result, name, value);
|
|
25
|
+
continue;
|
|
26
|
+
}
|
|
27
|
+
const name = arg.slice(2);
|
|
28
|
+
if (BOOLEAN_FLAGS.has(name)) {
|
|
29
|
+
applyFlag(result, name);
|
|
30
|
+
continue;
|
|
31
|
+
}
|
|
32
|
+
const nextArg = argv[i + 1];
|
|
33
|
+
if (VALUE_FLAGS.has(name) && nextArg !== undefined) {
|
|
34
|
+
i++;
|
|
35
|
+
applyFlag(result, name, nextArg);
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
return result;
|
|
39
|
+
}
|
|
40
|
+
function applyFlag(result, name, value) {
|
|
41
|
+
switch (name) {
|
|
42
|
+
case "help":
|
|
43
|
+
result.help = true;
|
|
44
|
+
break;
|
|
45
|
+
case "version":
|
|
46
|
+
result.version = true;
|
|
47
|
+
break;
|
|
48
|
+
case "force":
|
|
49
|
+
result.force = true;
|
|
50
|
+
break;
|
|
51
|
+
case "no-migrate":
|
|
52
|
+
result.noMigrate = true;
|
|
53
|
+
break;
|
|
54
|
+
case "config":
|
|
55
|
+
result.config = value;
|
|
56
|
+
break;
|
|
57
|
+
case "dialect":
|
|
58
|
+
result.dialect = value;
|
|
59
|
+
break;
|
|
60
|
+
case "host":
|
|
61
|
+
result.host = value;
|
|
62
|
+
break;
|
|
63
|
+
case "port":
|
|
64
|
+
result.port = value ? Number.parseInt(value, 10) : undefined;
|
|
65
|
+
break;
|
|
66
|
+
case "database":
|
|
67
|
+
result.database = value;
|
|
68
|
+
break;
|
|
69
|
+
case "user":
|
|
70
|
+
result.user = value;
|
|
71
|
+
break;
|
|
72
|
+
case "password":
|
|
73
|
+
result.password = value;
|
|
74
|
+
break;
|
|
75
|
+
case "tables":
|
|
76
|
+
result.tables = value?.split(",");
|
|
77
|
+
break;
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
//# sourceMappingURL=args.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"args.js","sourceRoot":"","sources":["../../src/cli/args.ts"],"names":[],"mappings":"AAgBA,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC,CAAC;AAC7C,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,YAAY,CAAC,CAAC,CAAC;AAC1E,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,CAAC,QAAQ,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,UAAU,EAAE,QAAQ,CAAC,CAAC,CAAC;AAE7G,MAAM,UAAU,SAAS,CAAC,IAAc;IACtC,MAAM,MAAM,GAAe;QACzB,OAAO,EAAE,KAAK;QACd,IAAI,EAAE,KAAK;QACX,OAAO,EAAE,KAAK;QACd,KAAK,EAAE,KAAK;QACZ,SAAS,EAAE,KAAK;KACjB,CAAC;IAEF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACrC,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;QACpB,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YACtD,MAAM,CAAC,OAAO,GAAG,GAAwB,CAAC;YAC1C,SAAS;QACX,CAAC;QACD,IAAI,CAAC,GAAG,EAAE,UAAU,CAAC,IAAI,CAAC;YAAE,SAAS;QAErC,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YACtB,MAAM,OAAO,GAAG,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;YACjC,MAAM,IAAI,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;YACnC,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC;YACrC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;YAC/B,SAAS;QACX,CAAC;QAED,MAAM,IAAI,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAE1B,IAAI,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YAC5B,SAAS,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;YACxB,SAAS;QACX,CAAC;QAED,MAAM,OAAO,GAAG,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC5B,IAAI,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;YACnD,CAAC,EAAE,CAAC;YACJ,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;QACnC,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,SAAS,CAAC,MAAkB,EAAE,IAAY,EAAE,KAAc;IACjE,QAAQ,IAAI,EAAE,CAAC;QACb,KAAK,MAAM;YACT,MAAM,CAAC,IAAI,GAAG,IAAI,CAAC;YACnB,MAAM;QACR,KAAK,SAAS;YACZ,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC;YACtB,MAAM;QACR,KAAK,OAAO;YACV,MAAM,CAAC,KAAK,GAAG,IAAI,CAAC;YACpB,MAAM;QACR,KAAK,YAAY;YACf,MAAM,CAAC,SAAS,GAAG,IAAI,CAAC;YACxB,MAAM;QACR,KAAK,QAAQ;YACX,MAAM,CAAC,MAAM,GAAG,KAAK,CAAC;YACtB,MAAM;QACR,KAAK,SAAS;YACZ,MAAM,CAAC,OAAO,GAAG,KAAK,CAAC;YACvB,MAAM;QACR,KAAK,MAAM;YACT,MAAM,CAAC,IAAI,GAAG,KAAK,CAAC;YACpB,MAAM;QACR,KAAK,MAAM;YACT,MAAM,CAAC,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;YAC7D,MAAM;QACR,KAAK,UAAU;YACb,MAAM,CAAC,QAAQ,GAAG,KAAK,CAAC;YACxB,MAAM;QACR,KAAK,MAAM;YACT,MAAM,CAAC,IAAI,GAAG,KAAK,CAAC;YACpB,MAAM;QACR,KAAK,UAAU;YACb,MAAM,CAAC,QAAQ,GAAG,KAAK,CAAC;YACxB,MAAM;QACR,KAAK,QAAQ;YACX,MAAM,CAAC,MAAM,GAAG,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC;YAClC,MAAM;IACV,CAAC;AACH,CAAC"}
|
package/dist/cli.d.ts
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":""}
|