zerobucket 0.11.0 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +46 -32
- package/dist/bin.js +25 -5
- package/dist/index.cjs +25 -5
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +5 -1
- package/dist/index.d.ts +5 -1
- package/dist/index.js +25 -5
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -339,7 +339,9 @@ What is different from Postgres, and why (each verified against a real MariaDB 1
|
|
|
339
339
|
`deleteMany()` run `SELECT ... FOR UPDATE` and then `DELETE` in one transaction. That is also how a tiered row's object
|
|
340
340
|
gets cleaned up.
|
|
341
341
|
- **Timestamps are written as UTC strings** (`DATETIME(6)`, naive UTC, like the Python package), so your server's or your
|
|
342
|
-
process's timezone setting cannot change what is stored.
|
|
342
|
+
process's timezone setting cannot change what is stored. The age filter behind `olderThanDays` and `tier --older-than`
|
|
343
|
+
compares against `UTC_TIMESTAMP()`, never `NOW()`; since 0.11.1 a test runs it with the session time zone at `+13:00` and
|
|
344
|
+
`-12:00` (the old suites only ever ran on UTC servers, where the two agree).
|
|
343
345
|
- **Pooling.** `mysql2` opens connections on demand, so `poolMinSize` is accepted for parity but there is no minimum to hold.
|
|
344
346
|
`mysql2` has no acquire timeout, so `poolTimeoutMs` is implemented here: waiting longer than that for a free connection
|
|
345
347
|
throws a clear `StorageError`, which is **not** retried (retrying would only wait again).
|
|
@@ -373,9 +375,9 @@ What is different from Postgres, and why (each verified against a real MariaDB 1
|
|
|
373
375
|
skipped on your connection (it cannot know whether you will commit).
|
|
374
376
|
- **Streaming** uses ranged `SUBSTRING()` reads, so a delete mid-stream raises `StorageError` (like Postgres). Tiered images
|
|
375
377
|
stream with real S3 `Range` requests.
|
|
376
|
-
- **Tested on MariaDB 10.11 and MySQL 8.0
|
|
377
|
-
|
|
378
|
-
MariaDB 10.2.1+ or MySQL 8.0.16+: older servers accept the `CHECK` constraint syntax but ignore it.
|
|
378
|
+
- **Tested on MariaDB 10.11 and MySQL 8.0** (CI runs the MySQL, dedup and CLI test suites and the Node/Python conformance
|
|
379
|
+
harness on both; the schema comparison was also run by hand on 8.0.46). Other versions are expected to behave the same but have not
|
|
380
|
+
been run. Use MariaDB 10.2.1+ or MySQL 8.0.16+: older servers accept the `CHECK` constraint syntax but ignore it.
|
|
379
381
|
|
|
380
382
|
## Deduplication
|
|
381
383
|
|
|
@@ -409,6 +411,20 @@ migrate everything else, pass `{ skipTiered: true }` (CLI: `--skip-tiered`): tie
|
|
|
409
411
|
classic table, counted in `tieredSkipped`, and **will not exist in the dedup tables**, so only use it if you can live
|
|
410
412
|
without them there.
|
|
411
413
|
|
|
414
|
+
It runs under a lock on every database (PostgreSQL advisory lock, SQLite write lock, MySQL/MariaDB named lock), so two
|
|
415
|
+
migrations started at once cannot race, and it keeps both `created_at` and `updated_at` of every row and reports one
|
|
416
|
+
`migrate_to_dedup` event to `onOperation`. (Before 0.12.0 the PostgreSQL migration kept only `created_at`, so migrated rows'
|
|
417
|
+
`updated_at` was the time of the migration, and it reported no event.) **Migrated on PostgreSQL before 0.12.0?** The classic
|
|
418
|
+
table is never modified, so it still holds the original `updated_at` of every image; this statement copies it over and is
|
|
419
|
+
safe to run more than once:
|
|
420
|
+
|
|
421
|
+
```sql
|
|
422
|
+
UPDATE zerobucket_image_refs r
|
|
423
|
+
SET updated_at = c.updated_at
|
|
424
|
+
FROM zerobucket_images c
|
|
425
|
+
WHERE c.id = r.id;
|
|
426
|
+
```
|
|
427
|
+
|
|
412
428
|
## Object-storage tiering
|
|
413
429
|
|
|
414
430
|
For the minority of apps whose image volume outgrows what is economical to keep in a database
|
|
@@ -457,31 +473,28 @@ same corpus of valid and corrupt files identically.
|
|
|
457
473
|
|
|
458
474
|
## Differences from the Python package
|
|
459
475
|
|
|
460
|
-
Python column = zerobucket on PyPI **0.
|
|
461
|
-
0.27.0 to 0.
|
|
462
|
-
|
|
463
|
-
| | Python
|
|
464
|
-
| -------------------------------------------------------------- |
|
|
465
|
-
| API | sync + async clients
|
|
466
|
-
| Async
|
|
467
|
-
| `optimize` | Pillow; EXIF stripped, not applied
|
|
468
|
-
| JPEG encoder in `optimize` | Pillow, progressive + optimized
|
|
469
|
-
| JPEG missing only its last 1-2 bytes (the end-of-image marker) | accepted (Pillow reads every pixel)
|
|
470
|
-
| `putMany` inserts | pipelined
|
|
471
|
-
| Formats | JPEG, PNG, WebP, HEIC (`[heic]` extra)
|
|
472
|
-
| Convert TO HEIC with `optimize` | yes (pillow-heif)
|
|
473
|
-
| Event duration | `duration_seconds`
|
|
474
|
-
| `migrateClassicToDedup`, tiered classic rows | always **skips** them and reports `tiered_rows_skipped`
|
|
475
|
-
|
|
|
476
|
-
| `
|
|
477
|
-
| SQLite
|
|
478
|
-
|
|
|
479
|
-
|
|
|
480
|
-
|
|
|
481
|
-
|
|
|
482
|
-
| `connection=` without an open transaction | multi-statement ops rely on the driver's implicit transaction | always atomic (own transaction or savepoint) |
|
|
483
|
-
| S3 delete failure after `delete()` | never fails the delete; reported as an `object_storage_delete` event (since 0.29.0) | never fails the delete; reported via `onOperation` |
|
|
484
|
-
| Options | snake_case kwargs | camelCase options object |
|
|
476
|
+
Python column = zerobucket on PyPI **0.34.0 or later** (several of its behaviours below were fixed or added in 0.25.1, 0.26.0 and
|
|
477
|
+
0.27.0 to 0.34.0; older versions differ).
|
|
478
|
+
|
|
479
|
+
| | Python | npm |
|
|
480
|
+
| -------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------- |
|
|
481
|
+
| API | sync + async clients | one async client |
|
|
482
|
+
| Async client feature set | everything the sync client has: dedup and tiering (since 0.30.0), hooks, `optimize` and `validator` (since 0.32.0), `connection=` (since 0.33.0); a hook may be a plain or an `async def` function | one async client, always, with all of them |
|
|
483
|
+
| `optimize` | Pillow; EXIF stripped, not applied | sharp; EXIF **applied then stripped**; transparency → **white** for JPEG |
|
|
484
|
+
| JPEG encoder in `optimize` | Pillow, progressive + optimized | sharp with `mozjpeg`, progressive: on smooth photos roughly **half the size** at about the same SSIM (measured); WebP and PNG match |
|
|
485
|
+
| JPEG missing only its last 1-2 bytes (the end-of-image marker) | accepted (Pillow reads every pixel) | **rejected** (libvips treats a premature end as an error); truncated interlaced PNGs are rejected by both since npm 0.10.0 |
|
|
486
|
+
| `putMany` inserts | pipelined | one transaction, statements run sequentially |
|
|
487
|
+
| Formats | JPEG, PNG, WebP, HEIC (`[heic]` extra) | JPEG, PNG, WebP, HEIC (optional `libheif-js`) |
|
|
488
|
+
| Convert TO HEIC with `optimize` | yes (pillow-heif) | no (no HEVC encoder for Node); converting FROM HEIC works; `optimize: true` on a HEIC needs an explicit `format` |
|
|
489
|
+
| Event duration | `duration_seconds` | `durationMs` |
|
|
490
|
+
| `migrateClassicToDedup`, tiered classic rows | always **skips** them and reports `tiered_rows_skipped` | **refuses** by default (changes nothing); `skipTiered: true` / `--skip-tiered` skips and reports like Python |
|
|
491
|
+
| SQLite API | synchronous `SQLiteBackend` + separate async backend | one async client; the sync driver runs on the event loop, lock waits are async (never block it) |
|
|
492
|
+
| SQLite `onOperation` / busy retry | `on_operation` events and busy retry, sync and async (since 0.27.0) | `onOperation` events and async `SQLITE_BUSY` retry (`busyTimeoutMs`) |
|
|
493
|
+
| SQLite stream deleted mid-read | sync adapter keeps reading a WAL snapshot (async adapter raises) | raises `StorageError`, like Postgres; use `connection` in a transaction for a snapshot |
|
|
494
|
+
| MySQL `onOperation` / retry | yes, sync and async (since 0.27.0) | yes (same client for everything) |
|
|
495
|
+
| `connection=` without an open transaction | multi-statement ops rely on the driver's implicit transaction | always atomic (own transaction or savepoint) |
|
|
496
|
+
| S3 delete failure after `delete()` | never fails the delete; reported as an `object_storage_delete` event (since 0.29.0) | never fails the delete; reported via `onOperation` |
|
|
497
|
+
| Options | snake_case kwargs | camelCase options object |
|
|
485
498
|
|
|
486
499
|
Both packages treat a malformed id as "not found" on every backend (Python 0.26.0+, npm 0.9.0+) and never pass one to a
|
|
487
500
|
`before_get`/`beforeGet` hook. Both packages now leave a tiered image's object in place on `delete(id, connection=...)` (a rolled-back transaction can
|
|
@@ -528,13 +541,14 @@ memory, event-loop impact, throughput, and the before and after of each release)
|
|
|
528
541
|
[Operations Guide](https://github.com/KedarGhadyalji/ZeroBucket/blob/main/docs/OPERATIONS.md).
|
|
529
542
|
- **Dedup and tiering cannot be combined** (a shared blob has many owners): `dedup: true` with `objectStorage` throws.
|
|
530
543
|
- **Tiered classic images cannot be migrated** by `migrate --to-dedup` (see Deduplication for `skipTiered`). The migration itself
|
|
531
|
-
works on PostgreSQL, SQLite and MySQL/MariaDB since 0.11.0
|
|
544
|
+
works on PostgreSQL, SQLite and MySQL/MariaDB (npm since 0.11.0, the Python package since 0.31.0).
|
|
532
545
|
- **HEIC can be read and converted from, but not written.** There is no HEVC encoder for Node, so `optimize` cannot
|
|
533
546
|
produce HEIC.
|
|
534
547
|
- **SQLite allows one writer at a time**; lock waits are async and never block the event loop, but write-heavy
|
|
535
548
|
workloads belong on PostgreSQL or MySQL/MariaDB.
|
|
536
|
-
- **Tested versions:** CI runs PostgreSQL 16
|
|
537
|
-
|
|
549
|
+
- **Tested versions:** CI runs PostgreSQL 16, MariaDB 10.11 and MySQL 8.0 on Node 20, 22 and 24, and the SQLite and
|
|
550
|
+
filesystem tests on Windows (GitHub's Windows runners cannot run database servers, so PostgreSQL, MySQL and S3 are not
|
|
551
|
+
tested there). Other versions are expected to behave the same but have not been run.
|
|
538
552
|
|
|
539
553
|
## Runtime notes
|
|
540
554
|
|
package/dist/bin.js
CHANGED
|
@@ -879,7 +879,10 @@ var PostgresBackend = class {
|
|
|
879
879
|
* re-running is safe (already-migrated ids are skipped and reported, not an error). Classic rows
|
|
880
880
|
* already tiered to object storage are refused up front with a clear message (their bytes are
|
|
881
881
|
* not in the database to copy) unless `skipTiered: true`, which skips and counts them instead.
|
|
882
|
-
* Still all-or-nothing: one transaction
|
|
882
|
+
* Still all-or-nothing: one transaction, under an advisory lock (the Python package takes the same
|
|
883
|
+
* one since 0.34.0, so a Node and a Python migration of one database exclude each other).
|
|
884
|
+
* Both `created_at` and `updated_at` are kept exactly (before 0.12.0 `updated_at` became the time of
|
|
885
|
+
* the migration), and one `migrate_to_dedup` event is reported to `onOperation` (before 0.12.0, none).
|
|
883
886
|
*/
|
|
884
887
|
async migrateClassicToDedup(opts = {}) {
|
|
885
888
|
if (!this.dedup) {
|
|
@@ -887,6 +890,17 @@ var PostgresBackend = class {
|
|
|
887
890
|
"migrateClassicToDedup() must be called on an instance created with dedup: true."
|
|
888
891
|
);
|
|
889
892
|
}
|
|
893
|
+
const start = performance.now();
|
|
894
|
+
try {
|
|
895
|
+
const result = await this.migrateClassicToDedupInner(opts);
|
|
896
|
+
this.emit("migrate_to_dedup", start, true, null, 0);
|
|
897
|
+
return result;
|
|
898
|
+
} catch (exc) {
|
|
899
|
+
this.emit("migrate_to_dedup", start, false, msg(exc), 0);
|
|
900
|
+
throw exc;
|
|
901
|
+
}
|
|
902
|
+
}
|
|
903
|
+
async migrateClassicToDedupInner(opts) {
|
|
890
904
|
await this.ready();
|
|
891
905
|
const client = await this.pool.connect();
|
|
892
906
|
let destroy = false;
|
|
@@ -923,11 +937,11 @@ var PostgresBackend = class {
|
|
|
923
937
|
const BATCH = 20;
|
|
924
938
|
for (; ; ) {
|
|
925
939
|
const page = cursor ? await client.query(
|
|
926
|
-
`SELECT id, data, mime_type, original_filename, size_bytes, width, height, checksum_sha256, created_at::text AS created_at_txt
|
|
940
|
+
`SELECT id, data, mime_type, original_filename, size_bytes, width, height, checksum_sha256, created_at::text AS created_at_txt, updated_at::text AS updated_at_txt
|
|
927
941
|
FROM zerobucket_images WHERE (created_at, id) > ($1::timestamptz, $2::uuid)${onlyInDb ? " AND storage_backend = 'postgres'" : ""} ORDER BY created_at, id LIMIT ${BATCH}`,
|
|
928
942
|
[cursor.createdAt, cursor.id]
|
|
929
943
|
) : await client.query(
|
|
930
|
-
`SELECT id, data, mime_type, original_filename, size_bytes, width, height, checksum_sha256, created_at::text AS created_at_txt
|
|
944
|
+
`SELECT id, data, mime_type, original_filename, size_bytes, width, height, checksum_sha256, created_at::text AS created_at_txt, updated_at::text AS updated_at_txt
|
|
931
945
|
FROM zerobucket_images${onlyInDb ? " WHERE storage_backend = 'postgres'" : ""} ORDER BY created_at, id LIMIT ${BATCH}`
|
|
932
946
|
);
|
|
933
947
|
if (page.rows.length === 0) break;
|
|
@@ -966,8 +980,14 @@ var PostgresBackend = class {
|
|
|
966
980
|
}
|
|
967
981
|
for (const r of fresh) {
|
|
968
982
|
await client.query(
|
|
969
|
-
"INSERT INTO zerobucket_image_refs (id, checksum_sha256, original_filename, created_at) VALUES ($1, $2, $3, $4::timestamptz)",
|
|
970
|
-
[
|
|
983
|
+
"INSERT INTO zerobucket_image_refs (id, checksum_sha256, original_filename, created_at, updated_at) VALUES ($1, $2, $3, $4::timestamptz, $5::timestamptz)",
|
|
984
|
+
[
|
|
985
|
+
r.id,
|
|
986
|
+
r.checksum_sha256,
|
|
987
|
+
r.original_filename,
|
|
988
|
+
r.created_at_txt,
|
|
989
|
+
r.updated_at_txt
|
|
990
|
+
]
|
|
971
991
|
);
|
|
972
992
|
seen.add(r.checksum_sha256);
|
|
973
993
|
migrated += 1;
|
package/dist/index.cjs
CHANGED
|
@@ -984,7 +984,10 @@ var PostgresBackend = class {
|
|
|
984
984
|
* re-running is safe (already-migrated ids are skipped and reported, not an error). Classic rows
|
|
985
985
|
* already tiered to object storage are refused up front with a clear message (their bytes are
|
|
986
986
|
* not in the database to copy) unless `skipTiered: true`, which skips and counts them instead.
|
|
987
|
-
* Still all-or-nothing: one transaction
|
|
987
|
+
* Still all-or-nothing: one transaction, under an advisory lock (the Python package takes the same
|
|
988
|
+
* one since 0.34.0, so a Node and a Python migration of one database exclude each other).
|
|
989
|
+
* Both `created_at` and `updated_at` are kept exactly (before 0.12.0 `updated_at` became the time of
|
|
990
|
+
* the migration), and one `migrate_to_dedup` event is reported to `onOperation` (before 0.12.0, none).
|
|
988
991
|
*/
|
|
989
992
|
async migrateClassicToDedup(opts = {}) {
|
|
990
993
|
if (!this.dedup) {
|
|
@@ -992,6 +995,17 @@ var PostgresBackend = class {
|
|
|
992
995
|
"migrateClassicToDedup() must be called on an instance created with dedup: true."
|
|
993
996
|
);
|
|
994
997
|
}
|
|
998
|
+
const start = performance.now();
|
|
999
|
+
try {
|
|
1000
|
+
const result = await this.migrateClassicToDedupInner(opts);
|
|
1001
|
+
this.emit("migrate_to_dedup", start, true, null, 0);
|
|
1002
|
+
return result;
|
|
1003
|
+
} catch (exc) {
|
|
1004
|
+
this.emit("migrate_to_dedup", start, false, msg(exc), 0);
|
|
1005
|
+
throw exc;
|
|
1006
|
+
}
|
|
1007
|
+
}
|
|
1008
|
+
async migrateClassicToDedupInner(opts) {
|
|
995
1009
|
await this.ready();
|
|
996
1010
|
const client = await this.pool.connect();
|
|
997
1011
|
let destroy = false;
|
|
@@ -1028,11 +1042,11 @@ var PostgresBackend = class {
|
|
|
1028
1042
|
const BATCH = 20;
|
|
1029
1043
|
for (; ; ) {
|
|
1030
1044
|
const page = cursor ? await client.query(
|
|
1031
|
-
`SELECT id, data, mime_type, original_filename, size_bytes, width, height, checksum_sha256, created_at::text AS created_at_txt
|
|
1045
|
+
`SELECT id, data, mime_type, original_filename, size_bytes, width, height, checksum_sha256, created_at::text AS created_at_txt, updated_at::text AS updated_at_txt
|
|
1032
1046
|
FROM zerobucket_images WHERE (created_at, id) > ($1::timestamptz, $2::uuid)${onlyInDb ? " AND storage_backend = 'postgres'" : ""} ORDER BY created_at, id LIMIT ${BATCH}`,
|
|
1033
1047
|
[cursor.createdAt, cursor.id]
|
|
1034
1048
|
) : await client.query(
|
|
1035
|
-
`SELECT id, data, mime_type, original_filename, size_bytes, width, height, checksum_sha256, created_at::text AS created_at_txt
|
|
1049
|
+
`SELECT id, data, mime_type, original_filename, size_bytes, width, height, checksum_sha256, created_at::text AS created_at_txt, updated_at::text AS updated_at_txt
|
|
1036
1050
|
FROM zerobucket_images${onlyInDb ? " WHERE storage_backend = 'postgres'" : ""} ORDER BY created_at, id LIMIT ${BATCH}`
|
|
1037
1051
|
);
|
|
1038
1052
|
if (page.rows.length === 0) break;
|
|
@@ -1071,8 +1085,14 @@ var PostgresBackend = class {
|
|
|
1071
1085
|
}
|
|
1072
1086
|
for (const r of fresh) {
|
|
1073
1087
|
await client.query(
|
|
1074
|
-
"INSERT INTO zerobucket_image_refs (id, checksum_sha256, original_filename, created_at) VALUES ($1, $2, $3, $4::timestamptz)",
|
|
1075
|
-
[
|
|
1088
|
+
"INSERT INTO zerobucket_image_refs (id, checksum_sha256, original_filename, created_at, updated_at) VALUES ($1, $2, $3, $4::timestamptz, $5::timestamptz)",
|
|
1089
|
+
[
|
|
1090
|
+
r.id,
|
|
1091
|
+
r.checksum_sha256,
|
|
1092
|
+
r.original_filename,
|
|
1093
|
+
r.created_at_txt,
|
|
1094
|
+
r.updated_at_txt
|
|
1095
|
+
]
|
|
1076
1096
|
);
|
|
1077
1097
|
seen.add(r.checksum_sha256);
|
|
1078
1098
|
migrated += 1;
|