posti 3.5.0 → 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/{LICENSE.md → LICENSE} +1 -1
- package/README.md +333 -112
- package/dist/cli/args.d.ts +17 -0
- package/dist/cli/args.d.ts.map +1 -0
- package/dist/cli/args.js +80 -0
- package/dist/cli/args.js.map +1 -0
- package/dist/cli.d.ts +3 -0
- package/dist/cli.d.ts.map +1 -0
- package/dist/cli.js +85 -0
- package/dist/cli.js.map +1 -0
- package/dist/config/defaults.d.ts +2 -0
- package/dist/config/defaults.d.ts.map +1 -0
- package/dist/config/defaults.js +2 -0
- package/dist/config/defaults.js.map +1 -0
- package/dist/config/index.d.ts +5 -0
- package/dist/config/index.d.ts.map +1 -0
- package/dist/config/index.js +97 -1
- package/dist/config/index.js.map +1 -0
- package/dist/config/paths.d.ts +4 -0
- package/dist/config/paths.d.ts.map +1 -0
- package/dist/config/paths.js +16 -0
- package/dist/config/paths.js.map +1 -0
- package/dist/config/schema.d.ts +27 -0
- package/dist/config/schema.d.ts.map +1 -0
- package/dist/config/schema.js +24 -0
- package/dist/config/schema.js.map +1 -0
- package/dist/db/dialects.d.ts +15 -0
- package/dist/db/dialects.d.ts.map +1 -0
- package/dist/db/dialects.js +270 -0
- package/dist/db/dialects.js.map +1 -0
- package/dist/db/index.d.ts +27 -0
- package/dist/db/index.d.ts.map +1 -0
- package/dist/db/index.js +49 -0
- package/dist/db/index.js.map +1 -0
- package/dist/db/migrate.d.ts +8 -0
- package/dist/db/migrate.d.ts.map +1 -0
- package/dist/db/migrate.js +58 -0
- package/dist/db/migrate.js.map +1 -0
- package/dist/db/migrations/001_initial_schema.d.ts +3 -0
- package/dist/db/migrations/001_initial_schema.d.ts.map +1 -0
- package/dist/db/migrations/001_initial_schema.js +17 -0
- package/dist/db/migrations/001_initial_schema.js.map +1 -0
- package/dist/db/migrations/index.d.ts +3 -0
- package/dist/db/migrations/index.d.ts.map +1 -0
- package/dist/db/migrations/index.js +3 -0
- package/dist/db/migrations/index.js.map +1 -0
- package/dist/db/operations.d.ts +13 -0
- package/dist/db/operations.d.ts.map +1 -0
- package/dist/db/operations.js +179 -0
- package/dist/db/operations.js.map +1 -0
- package/dist/fetcher/url-parser.d.ts +9 -0
- package/dist/fetcher/url-parser.d.ts.map +1 -0
- package/dist/fetcher/url-parser.js +19 -0
- package/dist/fetcher/url-parser.js.map +1 -0
- package/dist/index.d.ts +15 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +9 -1
- package/dist/index.js.map +1 -0
- package/dist/loader/index.d.ts +7 -0
- package/dist/loader/index.d.ts.map +1 -0
- package/dist/loader/index.js +145 -0
- package/dist/loader/index.js.map +1 -0
- package/dist/loader/state.d.ts +5 -0
- package/dist/loader/state.d.ts.map +1 -0
- package/dist/loader/state.js +26 -0
- package/dist/loader/state.js.map +1 -0
- package/dist/parser/field-definitions.d.ts +14 -0
- package/dist/parser/field-definitions.d.ts.map +1 -0
- package/dist/parser/field-definitions.js +91 -0
- package/dist/parser/field-definitions.js.map +1 -0
- package/dist/parser/file-converter.d.ts +2 -0
- package/dist/parser/file-converter.d.ts.map +1 -0
- package/dist/parser/file-converter.js +7 -0
- package/dist/parser/file-converter.js.map +1 -0
- package/dist/parser/line-parser.d.ts +4 -0
- package/dist/parser/line-parser.d.ts.map +1 -0
- package/dist/parser/line-parser.js +28 -0
- package/dist/parser/line-parser.js.map +1 -0
- package/dist/parser/unzipper.d.ts +2 -0
- package/dist/parser/unzipper.d.ts.map +1 -0
- package/dist/parser/unzipper.js +11 -0
- package/dist/parser/unzipper.js.map +1 -0
- package/dist/utils/chunks.d.ts +2 -0
- package/dist/utils/chunks.d.ts.map +1 -0
- package/dist/utils/chunks.js +10 -0
- package/dist/utils/chunks.js.map +1 -0
- package/dist/utils/logger.d.ts +7 -0
- package/dist/utils/logger.d.ts.map +1 -0
- package/dist/utils/logger.js +8 -0
- package/dist/utils/logger.js.map +1 -0
- package/dist/utils/progress.d.ts +14 -0
- package/dist/utils/progress.d.ts.map +1 -0
- package/dist/utils/progress.js +68 -0
- package/dist/utils/progress.js.map +1 -0
- package/dist/utils/time.d.ts +2 -0
- package/dist/utils/time.d.ts.map +1 -0
- package/dist/utils/time.js +14 -0
- package/dist/utils/time.js.map +1 -0
- package/package.json +76 -63
- package/CHANGELOG.md +0 -345
- package/dist/classes/Posti.js +0 -11
- package/dist/classes/Sequelize.js +0 -15
- package/dist/posti.js +0 -2
- package/dist/utils/index.js +0 -55
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import type BetterSqlite3 from "better-sqlite3";
|
|
2
|
+
import type { Pool as MysqlPool } from "mysql2/promise";
|
|
3
|
+
import type { Pool as PgPool } from "pg";
|
|
4
|
+
interface SqliteConnection {
|
|
5
|
+
dialect: "sqlite";
|
|
6
|
+
raw: BetterSqlite3.Database;
|
|
7
|
+
}
|
|
8
|
+
interface MysqlConnection {
|
|
9
|
+
dialect: "mysql" | "mariadb";
|
|
10
|
+
raw: MysqlPool;
|
|
11
|
+
}
|
|
12
|
+
interface PostgresConnection {
|
|
13
|
+
dialect: "postgres";
|
|
14
|
+
raw: PgPool;
|
|
15
|
+
}
|
|
16
|
+
export type DbConnection = SqliteConnection | MysqlConnection | PostgresConnection;
|
|
17
|
+
export declare function createConnection(config: {
|
|
18
|
+
dialect: "postgres" | "mysql" | "mariadb" | "sqlite";
|
|
19
|
+
host?: string;
|
|
20
|
+
port?: number;
|
|
21
|
+
user?: string;
|
|
22
|
+
password?: string;
|
|
23
|
+
database: string;
|
|
24
|
+
}): Promise<DbConnection>;
|
|
25
|
+
export declare function closeConnection(conn: DbConnection): Promise<void>;
|
|
26
|
+
export {};
|
|
27
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/db/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,aAAa,MAAM,gBAAgB,CAAC;AAChD,OAAO,KAAK,EAAE,IAAI,IAAI,SAAS,EAAE,MAAM,gBAAgB,CAAC;AACxD,OAAO,KAAK,EAAE,IAAI,IAAI,MAAM,EAAE,MAAM,IAAI,CAAC;AAEzC,UAAU,gBAAgB;IACxB,OAAO,EAAE,QAAQ,CAAC;IAClB,GAAG,EAAE,aAAa,CAAC,QAAQ,CAAC;CAC7B;AAED,UAAU,eAAe;IACvB,OAAO,EAAE,OAAO,GAAG,SAAS,CAAC;IAC7B,GAAG,EAAE,SAAS,CAAC;CAChB;AAED,UAAU,kBAAkB;IAC1B,OAAO,EAAE,UAAU,CAAC;IACpB,GAAG,EAAE,MAAM,CAAC;CACb;AAED,MAAM,MAAM,YAAY,GAAG,gBAAgB,GAAG,eAAe,GAAG,kBAAkB,CAAC;AAEnF,wBAAsB,gBAAgB,CAAC,MAAM,EAAE;IAC7C,OAAO,EAAE,UAAU,GAAG,OAAO,GAAG,SAAS,GAAG,QAAQ,CAAC;IACrD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;CAClB,GAAG,OAAO,CAAC,YAAY,CAAC,CAiCxB;AAED,wBAAsB,eAAe,CAAC,IAAI,EAAE,YAAY,GAAG,OAAO,CAAC,IAAI,CAAC,CAavE"}
|
package/dist/db/index.js
ADDED
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
export async function createConnection(config) {
|
|
2
|
+
switch (config.dialect) {
|
|
3
|
+
case "sqlite": {
|
|
4
|
+
const Database = (await import("better-sqlite3")).default;
|
|
5
|
+
const sqlite = new Database(config.database);
|
|
6
|
+
sqlite.pragma("journal_mode = WAL");
|
|
7
|
+
sqlite.pragma("foreign_keys = ON");
|
|
8
|
+
return { dialect: "sqlite", raw: sqlite };
|
|
9
|
+
}
|
|
10
|
+
case "mysql":
|
|
11
|
+
case "mariadb": {
|
|
12
|
+
const mysql2 = await import("mysql2/promise");
|
|
13
|
+
const pool = mysql2.createPool({
|
|
14
|
+
host: config.host,
|
|
15
|
+
port: config.port,
|
|
16
|
+
user: config.user,
|
|
17
|
+
password: config.password,
|
|
18
|
+
database: config.database,
|
|
19
|
+
});
|
|
20
|
+
return { dialect: config.dialect, raw: pool };
|
|
21
|
+
}
|
|
22
|
+
case "postgres": {
|
|
23
|
+
const pg = await import("pg");
|
|
24
|
+
const pool = new pg.default.Pool({
|
|
25
|
+
host: config.host,
|
|
26
|
+
port: config.port,
|
|
27
|
+
user: config.user,
|
|
28
|
+
password: config.password,
|
|
29
|
+
database: config.database,
|
|
30
|
+
});
|
|
31
|
+
return { dialect: "postgres", raw: pool };
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
export async function closeConnection(conn) {
|
|
36
|
+
switch (conn.dialect) {
|
|
37
|
+
case "sqlite":
|
|
38
|
+
conn.raw.close();
|
|
39
|
+
break;
|
|
40
|
+
case "mysql":
|
|
41
|
+
case "mariadb":
|
|
42
|
+
await conn.raw.end();
|
|
43
|
+
break;
|
|
44
|
+
case "postgres":
|
|
45
|
+
await conn.raw.end();
|
|
46
|
+
break;
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/db/index.ts"],"names":[],"mappings":"AAqBA,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,MAOtC;IACC,QAAQ,MAAM,CAAC,OAAO,EAAE,CAAC;QACvB,KAAK,QAAQ,EAAE,CAAC;YACd,MAAM,QAAQ,GAAG,CAAC,MAAM,MAAM,CAAC,gBAAgB,CAAC,CAAC,CAAC,OAAO,CAAC;YAC1D,MAAM,MAAM,GAAG,IAAI,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YAC7C,MAAM,CAAC,MAAM,CAAC,oBAAoB,CAAC,CAAC;YACpC,MAAM,CAAC,MAAM,CAAC,mBAAmB,CAAC,CAAC;YACnC,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC;QAC5C,CAAC;QACD,KAAK,OAAO,CAAC;QACb,KAAK,SAAS,EAAE,CAAC;YACf,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,gBAAgB,CAAC,CAAC;YAC9C,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC;gBAC7B,IAAI,EAAE,MAAM,CAAC,IAAI;gBACjB,IAAI,EAAE,MAAM,CAAC,IAAI;gBACjB,IAAI,EAAE,MAAM,CAAC,IAAI;gBACjB,QAAQ,EAAE,MAAM,CAAC,QAAQ;gBACzB,QAAQ,EAAE,MAAM,CAAC,QAAQ;aAC1B,CAAC,CAAC;YACH,OAAO,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC;QAChD,CAAC;QACD,KAAK,UAAU,EAAE,CAAC;YAChB,MAAM,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,CAAC;YAC9B,MAAM,IAAI,GAAG,IAAI,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC;gBAC/B,IAAI,EAAE,MAAM,CAAC,IAAI;gBACjB,IAAI,EAAE,MAAM,CAAC,IAAI;gBACjB,IAAI,EAAE,MAAM,CAAC,IAAI;gBACjB,QAAQ,EAAE,MAAM,CAAC,QAAQ;gBACzB,QAAQ,EAAE,MAAM,CAAC,QAAQ;aAC1B,CAAC,CAAC;YACH,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC;QAC5C,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,IAAkB;IACtD,QAAQ,IAAI,CAAC,OAAO,EAAE,CAAC;QACrB,KAAK,QAAQ;YACX,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC;YACjB,MAAM;QACR,KAAK,OAAO,CAAC;QACb,KAAK,SAAS;YACZ,MAAM,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC;YACrB,MAAM;QACR,KAAK,UAAU;YACb,MAAM,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC;YACrB,MAAM;IACV,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { PostiConfig } from "../config/schema.js";
|
|
2
|
+
import type { DbConnection } from "./index.js";
|
|
3
|
+
export interface Migration {
|
|
4
|
+
name: string;
|
|
5
|
+
up: (db: DbConnection, config: PostiConfig) => Promise<void>;
|
|
6
|
+
}
|
|
7
|
+
export declare function runMigrations(db: DbConnection, config: PostiConfig, migrations?: Migration[]): Promise<void>;
|
|
8
|
+
//# sourceMappingURL=migrate.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"migrate.d.ts","sourceRoot":"","sources":["../../src/db/migrate.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAEvD,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAI/C,MAAM,WAAW,SAAS;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,EAAE,EAAE,CAAC,EAAE,EAAE,YAAY,EAAE,MAAM,EAAE,WAAW,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CAC9D;AAED,wBAAsB,aAAa,CACjC,EAAE,EAAE,YAAY,EAChB,MAAM,EAAE,WAAW,EACnB,UAAU,GAAE,SAAS,EAAsB,GAC1C,OAAO,CAAC,IAAI,CAAC,CAiBf"}
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import { logger } from "../utils/logger.js";
|
|
2
|
+
import { migrations as defaultMigrations } from "./migrations/index.js";
|
|
3
|
+
import { execSQL, queryRows, quoteIdentifier } from "./operations.js";
|
|
4
|
+
export async function runMigrations(db, config, migrations = defaultMigrations) {
|
|
5
|
+
await ensureMigrationsTable(db, config);
|
|
6
|
+
const applied = await getAppliedMigrations(db, config);
|
|
7
|
+
let count = 0;
|
|
8
|
+
for (const migration of migrations) {
|
|
9
|
+
if (applied.has(migration.name))
|
|
10
|
+
continue;
|
|
11
|
+
logger.info(`Applying migration: ${migration.name}`);
|
|
12
|
+
await migration.up(db, config);
|
|
13
|
+
await recordMigration(db, config, migration.name);
|
|
14
|
+
count++;
|
|
15
|
+
}
|
|
16
|
+
if (count > 0) {
|
|
17
|
+
logger.info(`Applied ${count} migration(s)`);
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
async function ensureMigrationsTable(db, config) {
|
|
21
|
+
const tableName = quoteIdentifier(db.dialect, `${config.tablePrefix}schema_migrations`);
|
|
22
|
+
const colName = quoteIdentifier(db.dialect, "name");
|
|
23
|
+
const colAppliedAt = quoteIdentifier(db.dialect, "applied_at");
|
|
24
|
+
let timestampDefault;
|
|
25
|
+
switch (db.dialect) {
|
|
26
|
+
case "sqlite":
|
|
27
|
+
timestampDefault = "TIMESTAMP DEFAULT (datetime('now'))";
|
|
28
|
+
break;
|
|
29
|
+
case "mysql":
|
|
30
|
+
case "mariadb":
|
|
31
|
+
timestampDefault = "TIMESTAMP DEFAULT CURRENT_TIMESTAMP";
|
|
32
|
+
break;
|
|
33
|
+
case "postgres":
|
|
34
|
+
timestampDefault = "TIMESTAMP DEFAULT NOW()";
|
|
35
|
+
break;
|
|
36
|
+
}
|
|
37
|
+
await execSQL(db, `CREATE TABLE IF NOT EXISTS ${tableName} (
|
|
38
|
+
${colName} VARCHAR(255) PRIMARY KEY,
|
|
39
|
+
${colAppliedAt} ${timestampDefault}
|
|
40
|
+
)`);
|
|
41
|
+
}
|
|
42
|
+
async function getAppliedMigrations(db, config) {
|
|
43
|
+
const tableName = quoteIdentifier(db.dialect, `${config.tablePrefix}schema_migrations`);
|
|
44
|
+
const colName = quoteIdentifier(db.dialect, "name");
|
|
45
|
+
const rows = await queryRows(db, `SELECT ${colName} FROM ${tableName}`);
|
|
46
|
+
return new Set(rows.map((r) => String(r.name)));
|
|
47
|
+
}
|
|
48
|
+
async function recordMigration(db, config, migrationName) {
|
|
49
|
+
const tableName = quoteIdentifier(db.dialect, `${config.tablePrefix}schema_migrations`);
|
|
50
|
+
const colName = quoteIdentifier(db.dialect, "name");
|
|
51
|
+
if (db.dialect === "postgres") {
|
|
52
|
+
await execSQL(db, `INSERT INTO ${tableName} (${colName}) VALUES ($1)`, [migrationName]);
|
|
53
|
+
}
|
|
54
|
+
else {
|
|
55
|
+
await execSQL(db, `INSERT INTO ${tableName} (${colName}) VALUES (?)`, [migrationName]);
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
//# sourceMappingURL=migrate.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"migrate.js","sourceRoot":"","sources":["../../src/db/migrate.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAE5C,OAAO,EAAE,UAAU,IAAI,iBAAiB,EAAE,MAAM,uBAAuB,CAAC;AACxE,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAOtE,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,EAAgB,EAChB,MAAmB,EACnB,UAAU,GAAgB,iBAAiB;IAE3C,MAAM,qBAAqB,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;IACxC,MAAM,OAAO,GAAG,MAAM,oBAAoB,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;IAEvD,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACnC,IAAI,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC;YAAE,SAAS;QAE1C,MAAM,CAAC,IAAI,CAAC,uBAAuB,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC;QACrD,MAAM,SAAS,CAAC,EAAE,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;QAC/B,MAAM,eAAe,CAAC,EAAE,EAAE,MAAM,EAAE,SAAS,CAAC,IAAI,CAAC,CAAC;QAClD,KAAK,EAAE,CAAC;IACV,CAAC;IAED,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;QACd,MAAM,CAAC,IAAI,CAAC,WAAW,KAAK,eAAe,CAAC,CAAC;IAC/C,CAAC;AACH,CAAC;AAED,KAAK,UAAU,qBAAqB,CAAC,EAAgB,EAAE,MAAmB;IACxE,MAAM,SAAS,GAAG,eAAe,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,MAAM,CAAC,WAAW,mBAAmB,CAAC,CAAC;IACxF,MAAM,OAAO,GAAG,eAAe,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IACpD,MAAM,YAAY,GAAG,eAAe,CAAC,EAAE,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;IAE/D,IAAI,gBAAwB,CAAC;IAC7B,QAAQ,EAAE,CAAC,OAAO,EAAE,CAAC;QACnB,KAAK,QAAQ;YACX,gBAAgB,GAAG,qCAAqC,CAAC;YACzD,MAAM;QACR,KAAK,OAAO,CAAC;QACb,KAAK,SAAS;YACZ,gBAAgB,GAAG,qCAAqC,CAAC;YACzD,MAAM;QACR,KAAK,UAAU;YACb,gBAAgB,GAAG,yBAAyB,CAAC;YAC7C,MAAM;IACV,CAAC;IAED,MAAM,OAAO,CACX,EAAE,EACF,8BAA8B,SAAS;IACvC,OAAO;IACP,YAAY,IAAI,gBAAgB;EAClC,CACC,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,oBAAoB,CAAC,EAAgB,EAAE,MAAmB;IACvE,MAAM,SAAS,GAAG,eAAe,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,MAAM,CAAC,WAAW,mBAAmB,CAAC,CAAC;IACxF,MAAM,OAAO,GAAG,eAAe,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IACpD,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,EAAE,EAAE,UAAU,OAAO,SAAS,SAAS,EAAE,CAAC,CAAC;IACxE,OAAO,IAAI,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClD,CAAC;AAED,KAAK,UAAU,eAAe,CAAC,EAAgB,EAAE,MAAmB,EAAE,aAAqB;IACzF,MAAM,SAAS,GAAG,eAAe,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,MAAM,CAAC,WAAW,mBAAmB,CAAC,CAAC;IACxF,MAAM,OAAO,GAAG,eAAe,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IAEpD,IAAI,EAAE,CAAC,OAAO,KAAK,UAAU,EAAE,CAAC;QAC9B,MAAM,OAAO,CAAC,EAAE,EAAE,eAAe,SAAS,KAAK,OAAO,eAAe,EAAE,CAAC,aAAa,CAAC,CAAC,CAAC;IAC1F,CAAC;SAAM,CAAC;QACN,MAAM,OAAO,CAAC,EAAE,EAAE,eAAe,SAAS,KAAK,OAAO,cAAc,EAAE,CAAC,aAAa,CAAC,CAAC,CAAC;IACzF,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"001_initial_schema.d.ts","sourceRoot":"","sources":["../../../src/db/migrations/001_initial_schema.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AAG/C,eAAO,MAAM,aAAa,EAAE,SAY3B,CAAC"}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import { getCreateTableSQL } from "../dialects.js";
|
|
2
|
+
import { execSQL } from "../operations.js";
|
|
3
|
+
export const initialSchema = {
|
|
4
|
+
name: "001_initial_schema",
|
|
5
|
+
up: async (db, config) => {
|
|
6
|
+
for (const table of config.tables) {
|
|
7
|
+
const tableName = `${config.tablePrefix}${table}`;
|
|
8
|
+
const sql = getCreateTableSQL(db.dialect, tableName, table);
|
|
9
|
+
for (const statement of sql.split(";\n")) {
|
|
10
|
+
const trimmed = statement.trim();
|
|
11
|
+
if (trimmed)
|
|
12
|
+
await execSQL(db, trimmed);
|
|
13
|
+
}
|
|
14
|
+
}
|
|
15
|
+
},
|
|
16
|
+
};
|
|
17
|
+
//# sourceMappingURL=001_initial_schema.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"001_initial_schema.js","sourceRoot":"","sources":["../../../src/db/migrations/001_initial_schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AAEnD,OAAO,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC;AAE3C,MAAM,CAAC,MAAM,aAAa,GAAc;IACtC,IAAI,EAAE,oBAAoB;IAC1B,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE;QACvB,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;YAClC,MAAM,SAAS,GAAG,GAAG,MAAM,CAAC,WAAW,GAAG,KAAK,EAAE,CAAC;YAClD,MAAM,GAAG,GAAG,iBAAiB,CAAC,EAAE,CAAC,OAAO,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC;YAC5D,KAAK,MAAM,SAAS,IAAI,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CAAC;gBACzC,MAAM,OAAO,GAAG,SAAS,CAAC,IAAI,EAAE,CAAC;gBACjC,IAAI,OAAO;oBAAE,MAAM,OAAO,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;YAC1C,CAAC;QACH,CAAC;IACH,CAAC;CACF,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/db/migrations/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AAG/C,eAAO,MAAM,UAAU,EAAE,SAAS,EAAoB,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/db/migrations/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,aAAa,EAAE,MAAM,yBAAyB,CAAC;AAExD,MAAM,CAAC,MAAM,UAAU,GAAgB,CAAC,aAAa,CAAC,CAAC"}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import type { PostiConfig } from "../config/schema.js";
|
|
2
|
+
import { type SqlValue } from "./dialects.js";
|
|
3
|
+
import type { DbConnection } from "./index.js";
|
|
4
|
+
export declare function dropTables(db: DbConnection, config: PostiConfig): Promise<void>;
|
|
5
|
+
export declare function dropTable(db: DbConnection, tableName: string): Promise<void>;
|
|
6
|
+
export declare function createTempTable(db: DbConnection, tempTableName: string, sourceTableName: string): Promise<void>;
|
|
7
|
+
export declare function insertRows(db: DbConnection, tableName: string, rows: Record<string, SqlValue>[], chunkSize?: number): Promise<void>;
|
|
8
|
+
export declare function upsertFromTemp(db: DbConnection, dialect: PostiConfig["dialect"], tempTableName: string, finalTableName: string, tableType: string): Promise<void>;
|
|
9
|
+
export declare function countRows(db: DbConnection, tableName: string): Promise<number>;
|
|
10
|
+
export declare function quoteIdentifier(dialect: PostiConfig["dialect"], name: string): string;
|
|
11
|
+
export declare function execSQL(db: DbConnection, sql: string, values?: SqlValue[]): Promise<void>;
|
|
12
|
+
export declare function queryRows(db: DbConnection, sql: string, values?: SqlValue[]): Promise<Record<string, unknown>[]>;
|
|
13
|
+
//# sourceMappingURL=operations.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"operations.d.ts","sourceRoot":"","sources":["../../src/db/operations.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAEvD,OAAO,EAA8B,KAAK,QAAQ,EAAE,MAAM,eAAe,CAAC;AAC1E,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAiD/C,wBAAsB,UAAU,CAAC,EAAE,EAAE,YAAY,EAAE,MAAM,EAAE,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,CAKrF;AAED,wBAAsB,SAAS,CAAC,EAAE,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAElF;AAED,wBAAsB,eAAe,CAAC,EAAE,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,EAAE,eAAe,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAoBrH;AAED,wBAAsB,UAAU,CAC9B,EAAE,EAAE,YAAY,EAChB,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,QAAQ,CAAC,EAAE,EAChC,SAAS,SAAO,GACf,OAAO,CAAC,IAAI,CAAC,CAUf;AAED,wBAAsB,cAAc,CAClC,EAAE,EAAE,YAAY,EAChB,OAAO,EAAE,WAAW,CAAC,SAAS,CAAC,EAC/B,aAAa,EAAE,MAAM,EACrB,cAAc,EAAE,MAAM,EACtB,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC,IAAI,CAAC,CAKf;AAED,wBAAsB,SAAS,CAAC,EAAE,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAmBpF;AAWD,wBAAgB,eAAe,CAAC,OAAO,EAAE,WAAW,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,CASrF;AAED,wBAAsB,OAAO,CAAC,EAAE,EAAE,YAAY,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAiB/F;AAED,wBAAsB,SAAS,CAC7B,EAAE,EAAE,YAAY,EAChB,GAAG,EAAE,MAAM,EACX,MAAM,CAAC,EAAE,QAAQ,EAAE,GAClB,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC,CAcpC"}
|
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
import { sliceIntoChunks } from "../utils/chunks.js";
|
|
2
|
+
import { getInsertSQL, getUpsertSQL } from "./dialects.js";
|
|
3
|
+
async function withTransaction(db, fn) {
|
|
4
|
+
switch (db.dialect) {
|
|
5
|
+
case "sqlite": {
|
|
6
|
+
// SQLite better-sqlite3 transactions are synchronous, but our operations
|
|
7
|
+
// use the sync API internally for SQLite, so we manually wrap with BEGIN/COMMIT
|
|
8
|
+
try {
|
|
9
|
+
db.raw.exec("BEGIN");
|
|
10
|
+
const result = await fn();
|
|
11
|
+
db.raw.exec("COMMIT");
|
|
12
|
+
return result;
|
|
13
|
+
}
|
|
14
|
+
catch (error) {
|
|
15
|
+
db.raw.exec("ROLLBACK");
|
|
16
|
+
throw error;
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
case "mysql":
|
|
20
|
+
case "mariadb": {
|
|
21
|
+
const conn = await db.raw.getConnection();
|
|
22
|
+
try {
|
|
23
|
+
await conn.beginTransaction();
|
|
24
|
+
const result = await fn();
|
|
25
|
+
await conn.commit();
|
|
26
|
+
return result;
|
|
27
|
+
}
|
|
28
|
+
catch (error) {
|
|
29
|
+
await conn.rollback();
|
|
30
|
+
throw error;
|
|
31
|
+
}
|
|
32
|
+
finally {
|
|
33
|
+
conn.release();
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
case "postgres": {
|
|
37
|
+
const client = await db.raw.connect();
|
|
38
|
+
try {
|
|
39
|
+
await client.query("BEGIN");
|
|
40
|
+
const result = await fn();
|
|
41
|
+
await client.query("COMMIT");
|
|
42
|
+
return result;
|
|
43
|
+
}
|
|
44
|
+
catch (error) {
|
|
45
|
+
await client.query("ROLLBACK");
|
|
46
|
+
throw error;
|
|
47
|
+
}
|
|
48
|
+
finally {
|
|
49
|
+
client.release();
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
export async function dropTables(db, config) {
|
|
55
|
+
for (const table of config.tables) {
|
|
56
|
+
const tableName = `${config.tablePrefix}${table}`;
|
|
57
|
+
await dropTable(db, tableName);
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
export async function dropTable(db, tableName) {
|
|
61
|
+
await execSQL(db, `DROP TABLE IF EXISTS ${quoteIdentifier(db.dialect, tableName)}`);
|
|
62
|
+
}
|
|
63
|
+
export async function createTempTable(db, tempTableName, sourceTableName) {
|
|
64
|
+
await dropTable(db, tempTableName);
|
|
65
|
+
const quotedTemp = quoteIdentifier(db.dialect, tempTableName);
|
|
66
|
+
const quotedSource = quoteIdentifier(db.dialect, sourceTableName);
|
|
67
|
+
let sql;
|
|
68
|
+
switch (db.dialect) {
|
|
69
|
+
case "sqlite":
|
|
70
|
+
sql = `CREATE TABLE ${quotedTemp} AS SELECT * FROM ${quotedSource} WHERE 0`;
|
|
71
|
+
break;
|
|
72
|
+
case "mysql":
|
|
73
|
+
case "mariadb":
|
|
74
|
+
sql = `CREATE TABLE ${quotedTemp} LIKE ${quotedSource}`;
|
|
75
|
+
break;
|
|
76
|
+
case "postgres":
|
|
77
|
+
sql = `CREATE TABLE ${quotedTemp} (LIKE ${quotedSource} INCLUDING ALL)`;
|
|
78
|
+
break;
|
|
79
|
+
}
|
|
80
|
+
await execSQL(db, sql);
|
|
81
|
+
}
|
|
82
|
+
export async function insertRows(db, tableName, rows, chunkSize = 1000) {
|
|
83
|
+
if (rows.length === 0)
|
|
84
|
+
return;
|
|
85
|
+
await withTransaction(db, async () => {
|
|
86
|
+
const chunks = sliceIntoChunks(rows, chunkSize);
|
|
87
|
+
for (const chunk of chunks) {
|
|
88
|
+
const { sql, values } = getInsertSQL(db.dialect, tableName, chunk);
|
|
89
|
+
await execSQL(db, sql, values);
|
|
90
|
+
}
|
|
91
|
+
});
|
|
92
|
+
}
|
|
93
|
+
export async function upsertFromTemp(db, dialect, tempTableName, finalTableName, tableType) {
|
|
94
|
+
await withTransaction(db, async () => {
|
|
95
|
+
const sql = getUpsertSQL(dialect, tempTableName, finalTableName, tableType);
|
|
96
|
+
await execSQL(db, sql);
|
|
97
|
+
});
|
|
98
|
+
}
|
|
99
|
+
export async function countRows(db, tableName) {
|
|
100
|
+
const sql = `SELECT COUNT(*) as cnt FROM ${quoteIdentifier(db.dialect, tableName)}`;
|
|
101
|
+
switch (db.dialect) {
|
|
102
|
+
case "sqlite": {
|
|
103
|
+
const rows = db.raw.prepare(sql).all();
|
|
104
|
+
return extractCount(rows[0]);
|
|
105
|
+
}
|
|
106
|
+
case "mysql":
|
|
107
|
+
case "mariadb": {
|
|
108
|
+
const [rows] = await db.raw.execute(sql);
|
|
109
|
+
const first = Array.isArray(rows) ? rows[0] : undefined;
|
|
110
|
+
return extractCount(first);
|
|
111
|
+
}
|
|
112
|
+
case "postgres": {
|
|
113
|
+
const result = await db.raw.query(sql);
|
|
114
|
+
return extractCount(result.rows[0]);
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
function extractCount(row) {
|
|
119
|
+
if (row == null || typeof row !== "object")
|
|
120
|
+
return 0;
|
|
121
|
+
const cnt = row.cnt;
|
|
122
|
+
if (typeof cnt === "number")
|
|
123
|
+
return cnt;
|
|
124
|
+
if (typeof cnt === "string")
|
|
125
|
+
return Number.parseInt(cnt, 10) || 0;
|
|
126
|
+
if (typeof cnt === "bigint")
|
|
127
|
+
return Number(cnt);
|
|
128
|
+
return 0;
|
|
129
|
+
}
|
|
130
|
+
export function quoteIdentifier(dialect, name) {
|
|
131
|
+
switch (dialect) {
|
|
132
|
+
case "mysql":
|
|
133
|
+
case "mariadb":
|
|
134
|
+
return `\`${name.replaceAll("`", "``")}\``;
|
|
135
|
+
case "postgres":
|
|
136
|
+
case "sqlite":
|
|
137
|
+
return `"${name.replaceAll('"', '""')}"`;
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
export async function execSQL(db, sql, values) {
|
|
141
|
+
switch (db.dialect) {
|
|
142
|
+
case "sqlite":
|
|
143
|
+
if (values && values.length > 0) {
|
|
144
|
+
db.raw.prepare(sql).run(...values);
|
|
145
|
+
}
|
|
146
|
+
else {
|
|
147
|
+
db.raw.exec(sql);
|
|
148
|
+
}
|
|
149
|
+
break;
|
|
150
|
+
case "mysql":
|
|
151
|
+
case "mariadb":
|
|
152
|
+
await db.raw.execute(sql, values);
|
|
153
|
+
break;
|
|
154
|
+
case "postgres":
|
|
155
|
+
await db.raw.query(sql, values);
|
|
156
|
+
break;
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
export async function queryRows(db, sql, values) {
|
|
160
|
+
switch (db.dialect) {
|
|
161
|
+
case "sqlite":
|
|
162
|
+
return toRecordArray(db.raw.prepare(sql).all(...(values ?? [])));
|
|
163
|
+
case "mysql":
|
|
164
|
+
case "mariadb": {
|
|
165
|
+
const [rows] = await db.raw.execute(sql, values);
|
|
166
|
+
return toRecordArray(rows);
|
|
167
|
+
}
|
|
168
|
+
case "postgres": {
|
|
169
|
+
const result = await db.raw.query(sql, values);
|
|
170
|
+
return toRecordArray(result.rows);
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
function toRecordArray(value) {
|
|
175
|
+
if (!Array.isArray(value))
|
|
176
|
+
return [];
|
|
177
|
+
return value.filter((row) => typeof row === "object" && row !== null);
|
|
178
|
+
}
|
|
179
|
+
//# sourceMappingURL=operations.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"operations.js","sourceRoot":"","sources":["../../src/db/operations.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAE,YAAY,EAAE,YAAY,EAAiB,MAAM,eAAe,CAAC;AAG1E,KAAK,UAAU,eAAe,CAAI,EAAgB,EAAE,EAAoB;IACtE,QAAQ,EAAE,CAAC,OAAO,EAAE,CAAC;QACnB,KAAK,QAAQ,EAAE,CAAC;YACd,yEAAyE;YACzE,gFAAgF;YAChF,IAAI,CAAC;gBACH,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBACrB,MAAM,MAAM,GAAG,MAAM,EAAE,EAAE,CAAC;gBAC1B,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBACtB,OAAO,MAAM,CAAC;YAChB,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;gBACxB,MAAM,KAAK,CAAC;YACd,CAAC;QACH,CAAC;QACD,KAAK,OAAO,CAAC;QACb,KAAK,SAAS,EAAE,CAAC;YACf,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,GAAG,CAAC,aAAa,EAAE,CAAC;YAC1C,IAAI,CAAC;gBACH,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBAC9B,MAAM,MAAM,GAAG,MAAM,EAAE,EAAE,CAAC;gBAC1B,MAAM,IAAI,CAAC,MAAM,EAAE,CAAC;gBACpB,OAAO,MAAM,CAAC;YAChB,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACtB,MAAM,KAAK,CAAC;YACd,CAAC;oBAAS,CAAC;gBACT,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,CAAC;QACH,CAAC;QACD,KAAK,UAAU,EAAE,CAAC;YAChB,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;YACtC,IAAI,CAAC;gBACH,MAAM,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;gBAC5B,MAAM,MAAM,GAAG,MAAM,EAAE,EAAE,CAAC;gBAC1B,MAAM,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;gBAC7B,OAAO,MAAM,CAAC;YAChB,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;gBAC/B,MAAM,KAAK,CAAC;YACd,CAAC;oBAAS,CAAC;gBACT,MAAM,CAAC,OAAO,EAAE,CAAC;YACnB,CAAC;QACH,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,EAAgB,EAAE,MAAmB;IACpE,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;QAClC,MAAM,SAAS,GAAG,GAAG,MAAM,CAAC,WAAW,GAAG,KAAK,EAAE,CAAC;QAClD,MAAM,SAAS,CAAC,EAAE,EAAE,SAAS,CAAC,CAAC;IACjC,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,EAAgB,EAAE,SAAiB;IACjE,MAAM,OAAO,CAAC,EAAE,EAAE,wBAAwB,eAAe,CAAC,EAAE,CAAC,OAAO,EAAE,SAAS,CAAC,EAAE,CAAC,CAAC;AACtF,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,EAAgB,EAAE,aAAqB,EAAE,eAAuB;IACpG,MAAM,SAAS,CAAC,EAAE,EAAE,aAAa,CAAC,CAAC;IACnC,MAAM,UAAU,GAAG,eAAe,CAAC,EAAE,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;IAC9D,MAAM,YAAY,GAAG,eAAe,CAAC,EAAE,CAAC,OAAO,EAAE,eAAe,CAAC,CAAC;IAElE,IAAI,GAAW,CAAC;IAChB,QAAQ,EAAE,CAAC,OAAO,EAAE,CAAC;QACnB,KAAK,QAAQ;YACX,GAAG,GAAG,gBAAgB,UAAU,qBAAqB,YAAY,UAAU,CAAC;YAC5E,MAAM;QACR,KAAK,OAAO,CAAC;QACb,KAAK,SAAS;YACZ,GAAG,GAAG,gBAAgB,UAAU,SAAS,YAAY,EAAE,CAAC;YACxD,MAAM;QACR,KAAK,UAAU;YACb,GAAG,GAAG,gBAAgB,UAAU,UAAU,YAAY,iBAAiB,CAAC;YACxE,MAAM;IACV,CAAC;IAED,MAAM,OAAO,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;AACzB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,EAAgB,EAChB,SAAiB,EACjB,IAAgC,EAChC,SAAS,GAAG,IAAI;IAEhB,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO;IAE9B,MAAM,eAAe,CAAC,EAAE,EAAE,KAAK,IAAI,EAAE;QACnC,MAAM,MAAM,GAAG,eAAe,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;QAChD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,YAAY,CAAC,EAAE,CAAC,OAAO,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC;YACnE,MAAM,OAAO,CAAC,EAAE,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC;QACjC,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAClC,EAAgB,EAChB,OAA+B,EAC/B,aAAqB,EACrB,cAAsB,EACtB,SAAiB;IAEjB,MAAM,eAAe,CAAC,EAAE,EAAE,KAAK,IAAI,EAAE;QACnC,MAAM,GAAG,GAAG,YAAY,CAAC,OAAO,EAAE,aAAa,EAAE,cAAc,EAAE,SAAS,CAAC,CAAC;QAC5E,MAAM,OAAO,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;IACzB,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,EAAgB,EAAE,SAAiB;IACjE,MAAM,GAAG,GAAG,+BAA+B,eAAe,CAAC,EAAE,CAAC,OAAO,EAAE,SAAS,CAAC,EAAE,CAAC;IAEpF,QAAQ,EAAE,CAAC,OAAO,EAAE,CAAC;QACnB,KAAK,QAAQ,EAAE,CAAC;YACd,MAAM,IAAI,GAAG,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC;YACvC,OAAO,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QAC/B,CAAC;QACD,KAAK,OAAO,CAAC;QACb,KAAK,SAAS,EAAE,CAAC;YACf,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;YACzC,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;YACxD,OAAO,YAAY,CAAC,KAAK,CAAC,CAAC;QAC7B,CAAC;QACD,KAAK,UAAU,EAAE,CAAC;YAChB,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YACvC,OAAO,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QACtC,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,YAAY,CAAC,GAAY;IAChC,IAAI,GAAG,IAAI,IAAI,IAAI,OAAO,GAAG,KAAK,QAAQ;QAAE,OAAO,CAAC,CAAC;IACrD,MAAM,GAAG,GAAI,GAA+B,CAAC,GAAG,CAAC;IACjD,IAAI,OAAO,GAAG,KAAK,QAAQ;QAAE,OAAO,GAAG,CAAC;IACxC,IAAI,OAAO,GAAG,KAAK,QAAQ;QAAE,OAAO,MAAM,CAAC,QAAQ,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC;IAClE,IAAI,OAAO,GAAG,KAAK,QAAQ;QAAE,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC;IAChD,OAAO,CAAC,CAAC;AACX,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,OAA+B,EAAE,IAAY;IAC3E,QAAQ,OAAO,EAAE,CAAC;QAChB,KAAK,OAAO,CAAC;QACb,KAAK,SAAS;YACZ,OAAO,KAAK,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC;QAC7C,KAAK,UAAU,CAAC;QAChB,KAAK,QAAQ;YACX,OAAO,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC;IAC7C,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,EAAgB,EAAE,GAAW,EAAE,MAAmB;IAC9E,QAAQ,EAAE,CAAC,OAAO,EAAE,CAAC;QACnB,KAAK,QAAQ;YACX,IAAI,MAAM,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAChC,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC;YACrC,CAAC;iBAAM,CAAC;gBACN,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACnB,CAAC;YACD,MAAM;QACR,KAAK,OAAO,CAAC;QACb,KAAK,SAAS;YACZ,MAAM,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;YAClC,MAAM;QACR,KAAK,UAAU;YACb,MAAM,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;YAChC,MAAM;IACV,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,EAAgB,EAChB,GAAW,EACX,MAAmB;IAEnB,QAAQ,EAAE,CAAC,OAAO,EAAE,CAAC;QACnB,KAAK,QAAQ;YACX,OAAO,aAAa,CAAC,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QACnE,KAAK,OAAO,CAAC;QACb,KAAK,SAAS,EAAE,CAAC;YACf,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;YACjD,OAAO,aAAa,CAAC,IAAI,CAAC,CAAC;QAC7B,CAAC;QACD,KAAK,UAAU,EAAE,CAAC;YAChB,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;YAC/C,OAAO,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QACpC,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,aAAa,CAAC,KAAc;IACnC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IACrC,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,GAAG,EAAkC,EAAE,CAAC,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,KAAK,IAAI,CAAC,CAAC;AACxG,CAAC"}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { FileType } from "../parser/field-definitions.js";
|
|
2
|
+
export interface PostiFile {
|
|
3
|
+
url: string;
|
|
4
|
+
filename: string;
|
|
5
|
+
extensionless: string;
|
|
6
|
+
fileType: FileType;
|
|
7
|
+
}
|
|
8
|
+
export declare function parseFileUrls(html: string): PostiFile[];
|
|
9
|
+
//# sourceMappingURL=url-parser.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"url-parser.d.ts","sourceRoot":"","sources":["../../src/fetcher/url-parser.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,gCAAgC,CAAC;AAE/D,MAAM,WAAW,SAAS;IACxB,GAAG,EAAE,MAAM,CAAC;IACZ,QAAQ,EAAE,MAAM,CAAC;IACjB,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,QAAQ,CAAC;CACpB;AAQD,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,EAAE,CAgBvD"}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
const FILE_TYPES = new Set(["BAF", "PCF", "POM"]);
|
|
2
|
+
function isFileType(value) {
|
|
3
|
+
return FILE_TYPES.has(value);
|
|
4
|
+
}
|
|
5
|
+
export function parseFileUrls(html) {
|
|
6
|
+
const matches = html.matchAll(/href="((.*)?webpcode\/(((BAF|PCF|POM)(.*))\.zip))"/g);
|
|
7
|
+
const files = [];
|
|
8
|
+
for (const match of matches) {
|
|
9
|
+
const url = match[1];
|
|
10
|
+
const filename = match[3];
|
|
11
|
+
const extensionless = match[4];
|
|
12
|
+
const rawType = match[5];
|
|
13
|
+
if (url && filename && extensionless && rawType && isFileType(rawType)) {
|
|
14
|
+
files.push({ url, filename, extensionless, fileType: rawType });
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
return files;
|
|
18
|
+
}
|
|
19
|
+
//# sourceMappingURL=url-parser.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"url-parser.js","sourceRoot":"","sources":["../../src/fetcher/url-parser.ts"],"names":[],"mappings":"AASA,MAAM,UAAU,GAAG,IAAI,GAAG,CAAS,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC;AAE1D,SAAS,UAAU,CAAC,KAAa;IAC/B,OAAO,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;AAC/B,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,IAAY;IACxC,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,qDAAqD,CAAC,CAAC;IACrF,MAAM,KAAK,GAAgB,EAAE,CAAC;IAE9B,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,MAAM,GAAG,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACrB,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QAC1B,MAAM,aAAa,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QAC/B,MAAM,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QAEzB,IAAI,GAAG,IAAI,QAAQ,IAAI,aAAa,IAAI,OAAO,IAAI,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;YACvE,KAAK,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,QAAQ,EAAE,aAAa,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC;QAClE,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC"}
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
export { loadConfig } from "./config/index.js";
|
|
2
|
+
export type { PostiConfig } from "./config/schema.js";
|
|
3
|
+
export { configSchema } from "./config/schema.js";
|
|
4
|
+
export type { DbConnection } from "./db/index.js";
|
|
5
|
+
export { closeConnection, createConnection } from "./db/index.js";
|
|
6
|
+
export type { Migration } from "./db/migrate.js";
|
|
7
|
+
export { runMigrations } from "./db/migrate.js";
|
|
8
|
+
export type { PostiFile } from "./fetcher/url-parser.js";
|
|
9
|
+
export { parseFileUrls } from "./fetcher/url-parser.js";
|
|
10
|
+
export type { RunOptions } from "./loader/index.js";
|
|
11
|
+
export { run } from "./loader/index.js";
|
|
12
|
+
export type { FieldDefinition, FileType } from "./parser/field-definitions.js";
|
|
13
|
+
export { addressFields, fileTypeToFields, fileTypeToTable, zipcodeChangeFields, zipcodeFields, } from "./parser/field-definitions.js";
|
|
14
|
+
export { castValue, parseLine } from "./parser/line-parser.js";
|
|
15
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC/C,YAAY,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,YAAY,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAClD,OAAO,EAAE,eAAe,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AAClE,YAAY,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AACjD,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAChD,YAAY,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAC;AACzD,OAAO,EAAE,aAAa,EAAE,MAAM,yBAAyB,CAAC;AACxD,YAAY,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AACpD,OAAO,EAAE,GAAG,EAAE,MAAM,mBAAmB,CAAC;AACxC,YAAY,EAAE,eAAe,EAAE,QAAQ,EAAE,MAAM,+BAA+B,CAAC;AAC/E,OAAO,EACL,aAAa,EACb,gBAAgB,EAChB,eAAe,EACf,mBAAmB,EACnB,aAAa,GACd,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -1 +1,9 @@
|
|
|
1
|
-
|
|
1
|
+
export { loadConfig } from "./config/index.js";
|
|
2
|
+
export { configSchema } from "./config/schema.js";
|
|
3
|
+
export { closeConnection, createConnection } from "./db/index.js";
|
|
4
|
+
export { runMigrations } from "./db/migrate.js";
|
|
5
|
+
export { parseFileUrls } from "./fetcher/url-parser.js";
|
|
6
|
+
export { run } from "./loader/index.js";
|
|
7
|
+
export { addressFields, fileTypeToFields, fileTypeToTable, zipcodeChangeFields, zipcodeFields, } from "./parser/field-definitions.js";
|
|
8
|
+
export { castValue, parseLine } from "./parser/line-parser.js";
|
|
9
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAE/C,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAElD,OAAO,EAAE,eAAe,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AAElE,OAAO,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAEhD,OAAO,EAAE,aAAa,EAAE,MAAM,yBAAyB,CAAC;AAExD,OAAO,EAAE,GAAG,EAAE,MAAM,mBAAmB,CAAC;AAExC,OAAO,EACL,aAAa,EACb,gBAAgB,EAChB,eAAe,EACf,mBAAmB,EACnB,aAAa,GACd,MAAM,+BAA+B,CAAC;AACvC,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/loader/index.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAiBvD,MAAM,WAAW,UAAU;IACzB,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB,MAAM,CAAC,EAAE,OAAO,CAAC;CAClB;AAED,wBAAsB,GAAG,CAAC,MAAM,EAAE,WAAW,EAAE,OAAO,GAAE,UAAe,GAAG,OAAO,CAAC,IAAI,CAAC,CAsEtF"}
|