zerobucket 0.11.1 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +36 -23
- package/dist/bin.js +25 -5
- package/dist/index.cjs +25 -5
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +5 -1
- package/dist/index.d.ts +5 -1
- package/dist/index.js +25 -5
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -411,6 +411,20 @@ migrate everything else, pass `{ skipTiered: true }` (CLI: `--skip-tiered`): tie
|
|
|
411
411
|
classic table, counted in `tieredSkipped`, and **will not exist in the dedup tables**, so only use it if you can live
|
|
412
412
|
without them there.
|
|
413
413
|
|
|
414
|
+
It runs under a lock on every database (PostgreSQL advisory lock, SQLite write lock, MySQL/MariaDB named lock), so two
|
|
415
|
+
migrations started at once cannot race, and it keeps both `created_at` and `updated_at` of every row and reports one
|
|
416
|
+
`migrate_to_dedup` event to `onOperation`. (Before 0.12.0 the PostgreSQL migration kept only `created_at`, so migrated rows'
|
|
417
|
+
`updated_at` was the time of the migration, and it reported no event.) **Migrated on PostgreSQL before 0.12.0?** The classic
|
|
418
|
+
table is never modified, so it still holds the original `updated_at` of every image; this statement copies it over and is
|
|
419
|
+
safe to run more than once:
|
|
420
|
+
|
|
421
|
+
```sql
|
|
422
|
+
UPDATE zerobucket_image_refs r
|
|
423
|
+
SET updated_at = c.updated_at
|
|
424
|
+
FROM zerobucket_images c
|
|
425
|
+
WHERE c.id = r.id;
|
|
426
|
+
```
|
|
427
|
+
|
|
414
428
|
## Object-storage tiering
|
|
415
429
|
|
|
416
430
|
For the minority of apps whose image volume outgrows what is economical to keep in a database
|
|
@@ -459,29 +473,28 @@ same corpus of valid and corrupt files identically.
|
|
|
459
473
|
|
|
460
474
|
## Differences from the Python package
|
|
461
475
|
|
|
462
|
-
Python column = zerobucket on PyPI **0.
|
|
463
|
-
0.27.0 to 0.
|
|
464
|
-
|
|
465
|
-
| | Python
|
|
466
|
-
| -------------------------------------------------------------- |
|
|
467
|
-
| API | sync + async clients
|
|
468
|
-
| Async client feature set | everything the sync client has: dedup and tiering (since 0.30.0), hooks, `optimize` and `validator` (since 0.32.0), `connection=` (since 0.33.0); a hook may be a plain or an `async def` function
|
|
469
|
-
| `optimize` | Pillow; EXIF stripped, not applied
|
|
470
|
-
| JPEG encoder in `optimize` | Pillow, progressive + optimized
|
|
471
|
-
| JPEG missing only its last 1-2 bytes (the end-of-image marker) | accepted (Pillow reads every pixel)
|
|
472
|
-
| `putMany` inserts | pipelined
|
|
473
|
-
| Formats | JPEG, PNG, WebP, HEIC (`[heic]` extra)
|
|
474
|
-
| Convert TO HEIC with `optimize` | yes (pillow-heif)
|
|
475
|
-
| Event duration | `duration_seconds`
|
|
476
|
-
| `migrateClassicToDedup`, tiered classic rows | always **skips** them and reports `tiered_rows_skipped`
|
|
477
|
-
|
|
|
478
|
-
| SQLite
|
|
479
|
-
| SQLite
|
|
480
|
-
|
|
|
481
|
-
|
|
|
482
|
-
|
|
|
483
|
-
|
|
|
484
|
-
| Options | snake_case kwargs | camelCase options object |
|
|
476
|
+
Python column = zerobucket on PyPI **0.34.0 or later** (several of its behaviours below were fixed or added in 0.25.1, 0.26.0 and
|
|
477
|
+
0.27.0 to 0.34.0; older versions differ).
|
|
478
|
+
|
|
479
|
+
| | Python | npm |
|
|
480
|
+
| -------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------- |
|
|
481
|
+
| API | sync + async clients | one async client |
|
|
482
|
+
| Async client feature set | everything the sync client has: dedup and tiering (since 0.30.0), hooks, `optimize` and `validator` (since 0.32.0), `connection=` (since 0.33.0); a hook may be a plain or an `async def` function | one async client, always, with all of them |
|
|
483
|
+
| `optimize` | Pillow; EXIF stripped, not applied | sharp; EXIF **applied then stripped**; transparency → **white** for JPEG |
|
|
484
|
+
| JPEG encoder in `optimize` | Pillow, progressive + optimized | sharp with `mozjpeg`, progressive: on smooth photos roughly **half the size** at about the same SSIM (measured); WebP and PNG match |
|
|
485
|
+
| JPEG missing only its last 1-2 bytes (the end-of-image marker) | accepted (Pillow reads every pixel) | **rejected** (libvips treats a premature end as an error); truncated interlaced PNGs are rejected by both since npm 0.10.0 |
|
|
486
|
+
| `putMany` inserts | pipelined | one transaction, statements run sequentially |
|
|
487
|
+
| Formats | JPEG, PNG, WebP, HEIC (`[heic]` extra) | JPEG, PNG, WebP, HEIC (optional `libheif-js`) |
|
|
488
|
+
| Convert TO HEIC with `optimize` | yes (pillow-heif) | no (no HEVC encoder for Node); converting FROM HEIC works; `optimize: true` on a HEIC needs an explicit `format` |
|
|
489
|
+
| Event duration | `duration_seconds` | `durationMs` |
|
|
490
|
+
| `migrateClassicToDedup`, tiered classic rows | always **skips** them and reports `tiered_rows_skipped` | **refuses** by default (changes nothing); `skipTiered: true` / `--skip-tiered` skips and reports like Python |
|
|
491
|
+
| SQLite API | synchronous `SQLiteBackend` + separate async backend | one async client; the sync driver runs on the event loop, lock waits are async (never block it) |
|
|
492
|
+
| SQLite `onOperation` / busy retry | `on_operation` events and busy retry, sync and async (since 0.27.0) | `onOperation` events and async `SQLITE_BUSY` retry (`busyTimeoutMs`) |
|
|
493
|
+
| SQLite stream deleted mid-read | sync adapter keeps reading a WAL snapshot (async adapter raises) | raises `StorageError`, like Postgres; use `connection` in a transaction for a snapshot |
|
|
494
|
+
| MySQL `onOperation` / retry | yes, sync and async (since 0.27.0) | yes (same client for everything) |
|
|
495
|
+
| `connection=` without an open transaction | multi-statement ops rely on the driver's implicit transaction | always atomic (own transaction or savepoint) |
|
|
496
|
+
| S3 delete failure after `delete()` | never fails the delete; reported as an `object_storage_delete` event (since 0.29.0) | never fails the delete; reported via `onOperation` |
|
|
497
|
+
| Options | snake_case kwargs | camelCase options object |
|
|
485
498
|
|
|
486
499
|
Both packages treat a malformed id as "not found" on every backend (Python 0.26.0+, npm 0.9.0+) and never pass one to a
|
|
487
500
|
`before_get`/`beforeGet` hook. Both packages now leave a tiered image's object in place on `delete(id, connection=...)` (a rolled-back transaction can
|
package/dist/bin.js
CHANGED
|
@@ -879,7 +879,10 @@ var PostgresBackend = class {
|
|
|
879
879
|
* re-running is safe (already-migrated ids are skipped and reported, not an error). Classic rows
|
|
880
880
|
* already tiered to object storage are refused up front with a clear message (their bytes are
|
|
881
881
|
* not in the database to copy) unless `skipTiered: true`, which skips and counts them instead.
|
|
882
|
-
* Still all-or-nothing: one transaction
|
|
882
|
+
* Still all-or-nothing: one transaction, under an advisory lock (the Python package takes the same
|
|
883
|
+
* one since 0.34.0, so a Node and a Python migration of one database exclude each other).
|
|
884
|
+
* Both `created_at` and `updated_at` are kept exactly (before 0.12.0 `updated_at` became the time of
|
|
885
|
+
* the migration), and one `migrate_to_dedup` event is reported to `onOperation` (before 0.12.0, none).
|
|
883
886
|
*/
|
|
884
887
|
async migrateClassicToDedup(opts = {}) {
|
|
885
888
|
if (!this.dedup) {
|
|
@@ -887,6 +890,17 @@ var PostgresBackend = class {
|
|
|
887
890
|
"migrateClassicToDedup() must be called on an instance created with dedup: true."
|
|
888
891
|
);
|
|
889
892
|
}
|
|
893
|
+
const start = performance.now();
|
|
894
|
+
try {
|
|
895
|
+
const result = await this.migrateClassicToDedupInner(opts);
|
|
896
|
+
this.emit("migrate_to_dedup", start, true, null, 0);
|
|
897
|
+
return result;
|
|
898
|
+
} catch (exc) {
|
|
899
|
+
this.emit("migrate_to_dedup", start, false, msg(exc), 0);
|
|
900
|
+
throw exc;
|
|
901
|
+
}
|
|
902
|
+
}
|
|
903
|
+
async migrateClassicToDedupInner(opts) {
|
|
890
904
|
await this.ready();
|
|
891
905
|
const client = await this.pool.connect();
|
|
892
906
|
let destroy = false;
|
|
@@ -923,11 +937,11 @@ var PostgresBackend = class {
|
|
|
923
937
|
const BATCH = 20;
|
|
924
938
|
for (; ; ) {
|
|
925
939
|
const page = cursor ? await client.query(
|
|
926
|
-
`SELECT id, data, mime_type, original_filename, size_bytes, width, height, checksum_sha256, created_at::text AS created_at_txt
|
|
940
|
+
`SELECT id, data, mime_type, original_filename, size_bytes, width, height, checksum_sha256, created_at::text AS created_at_txt, updated_at::text AS updated_at_txt
|
|
927
941
|
FROM zerobucket_images WHERE (created_at, id) > ($1::timestamptz, $2::uuid)${onlyInDb ? " AND storage_backend = 'postgres'" : ""} ORDER BY created_at, id LIMIT ${BATCH}`,
|
|
928
942
|
[cursor.createdAt, cursor.id]
|
|
929
943
|
) : await client.query(
|
|
930
|
-
`SELECT id, data, mime_type, original_filename, size_bytes, width, height, checksum_sha256, created_at::text AS created_at_txt
|
|
944
|
+
`SELECT id, data, mime_type, original_filename, size_bytes, width, height, checksum_sha256, created_at::text AS created_at_txt, updated_at::text AS updated_at_txt
|
|
931
945
|
FROM zerobucket_images${onlyInDb ? " WHERE storage_backend = 'postgres'" : ""} ORDER BY created_at, id LIMIT ${BATCH}`
|
|
932
946
|
);
|
|
933
947
|
if (page.rows.length === 0) break;
|
|
@@ -966,8 +980,14 @@ var PostgresBackend = class {
|
|
|
966
980
|
}
|
|
967
981
|
for (const r of fresh) {
|
|
968
982
|
await client.query(
|
|
969
|
-
"INSERT INTO zerobucket_image_refs (id, checksum_sha256, original_filename, created_at) VALUES ($1, $2, $3, $4::timestamptz)",
|
|
970
|
-
[
|
|
983
|
+
"INSERT INTO zerobucket_image_refs (id, checksum_sha256, original_filename, created_at, updated_at) VALUES ($1, $2, $3, $4::timestamptz, $5::timestamptz)",
|
|
984
|
+
[
|
|
985
|
+
r.id,
|
|
986
|
+
r.checksum_sha256,
|
|
987
|
+
r.original_filename,
|
|
988
|
+
r.created_at_txt,
|
|
989
|
+
r.updated_at_txt
|
|
990
|
+
]
|
|
971
991
|
);
|
|
972
992
|
seen.add(r.checksum_sha256);
|
|
973
993
|
migrated += 1;
|
package/dist/index.cjs
CHANGED
|
@@ -984,7 +984,10 @@ var PostgresBackend = class {
|
|
|
984
984
|
* re-running is safe (already-migrated ids are skipped and reported, not an error). Classic rows
|
|
985
985
|
* already tiered to object storage are refused up front with a clear message (their bytes are
|
|
986
986
|
* not in the database to copy) unless `skipTiered: true`, which skips and counts them instead.
|
|
987
|
-
* Still all-or-nothing: one transaction
|
|
987
|
+
* Still all-or-nothing: one transaction, under an advisory lock (the Python package takes the same
|
|
988
|
+
* one since 0.34.0, so a Node and a Python migration of one database exclude each other).
|
|
989
|
+
* Both `created_at` and `updated_at` are kept exactly (before 0.12.0 `updated_at` became the time of
|
|
990
|
+
* the migration), and one `migrate_to_dedup` event is reported to `onOperation` (before 0.12.0, none).
|
|
988
991
|
*/
|
|
989
992
|
async migrateClassicToDedup(opts = {}) {
|
|
990
993
|
if (!this.dedup) {
|
|
@@ -992,6 +995,17 @@ var PostgresBackend = class {
|
|
|
992
995
|
"migrateClassicToDedup() must be called on an instance created with dedup: true."
|
|
993
996
|
);
|
|
994
997
|
}
|
|
998
|
+
const start = performance.now();
|
|
999
|
+
try {
|
|
1000
|
+
const result = await this.migrateClassicToDedupInner(opts);
|
|
1001
|
+
this.emit("migrate_to_dedup", start, true, null, 0);
|
|
1002
|
+
return result;
|
|
1003
|
+
} catch (exc) {
|
|
1004
|
+
this.emit("migrate_to_dedup", start, false, msg(exc), 0);
|
|
1005
|
+
throw exc;
|
|
1006
|
+
}
|
|
1007
|
+
}
|
|
1008
|
+
async migrateClassicToDedupInner(opts) {
|
|
995
1009
|
await this.ready();
|
|
996
1010
|
const client = await this.pool.connect();
|
|
997
1011
|
let destroy = false;
|
|
@@ -1028,11 +1042,11 @@ var PostgresBackend = class {
|
|
|
1028
1042
|
const BATCH = 20;
|
|
1029
1043
|
for (; ; ) {
|
|
1030
1044
|
const page = cursor ? await client.query(
|
|
1031
|
-
`SELECT id, data, mime_type, original_filename, size_bytes, width, height, checksum_sha256, created_at::text AS created_at_txt
|
|
1045
|
+
`SELECT id, data, mime_type, original_filename, size_bytes, width, height, checksum_sha256, created_at::text AS created_at_txt, updated_at::text AS updated_at_txt
|
|
1032
1046
|
FROM zerobucket_images WHERE (created_at, id) > ($1::timestamptz, $2::uuid)${onlyInDb ? " AND storage_backend = 'postgres'" : ""} ORDER BY created_at, id LIMIT ${BATCH}`,
|
|
1033
1047
|
[cursor.createdAt, cursor.id]
|
|
1034
1048
|
) : await client.query(
|
|
1035
|
-
`SELECT id, data, mime_type, original_filename, size_bytes, width, height, checksum_sha256, created_at::text AS created_at_txt
|
|
1049
|
+
`SELECT id, data, mime_type, original_filename, size_bytes, width, height, checksum_sha256, created_at::text AS created_at_txt, updated_at::text AS updated_at_txt
|
|
1036
1050
|
FROM zerobucket_images${onlyInDb ? " WHERE storage_backend = 'postgres'" : ""} ORDER BY created_at, id LIMIT ${BATCH}`
|
|
1037
1051
|
);
|
|
1038
1052
|
if (page.rows.length === 0) break;
|
|
@@ -1071,8 +1085,14 @@ var PostgresBackend = class {
|
|
|
1071
1085
|
}
|
|
1072
1086
|
for (const r of fresh) {
|
|
1073
1087
|
await client.query(
|
|
1074
|
-
"INSERT INTO zerobucket_image_refs (id, checksum_sha256, original_filename, created_at) VALUES ($1, $2, $3, $4::timestamptz)",
|
|
1075
|
-
[
|
|
1088
|
+
"INSERT INTO zerobucket_image_refs (id, checksum_sha256, original_filename, created_at, updated_at) VALUES ($1, $2, $3, $4::timestamptz, $5::timestamptz)",
|
|
1089
|
+
[
|
|
1090
|
+
r.id,
|
|
1091
|
+
r.checksum_sha256,
|
|
1092
|
+
r.original_filename,
|
|
1093
|
+
r.created_at_txt,
|
|
1094
|
+
r.updated_at_txt
|
|
1095
|
+
]
|
|
1076
1096
|
);
|
|
1077
1097
|
seen.add(r.checksum_sha256);
|
|
1078
1098
|
migrated += 1;
|