wavemind 2.2.9__tar.gz → 2.3.0__tar.gz

{wavemind-2.2.9 → wavemind-2.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: wavemind
-Version: 2.2.9
+Version: 2.3.0
 Summary: Local-first dynamic memory field with vector search and wave-field re-ranking
 License-Expression: MIT
 Project-URL: Homepage, https://github.com/CaspianG/wavemind
@@ -25,6 +25,8 @@ Provides-Extra: postgres
 Requires-Dist: psycopg[binary]>=3.1; extra == "postgres"
 Provides-Extra: redis
 Requires-Dist: redis>=5.0; extra == "redis"
+Provides-Extra: s3
+Requires-Dist: boto3>=1.34; extra == "s3"
 Provides-Extra: otel
 Requires-Dist: opentelemetry-api>=1.25; extra == "otel"
 Requires-Dist: opentelemetry-sdk>=1.25; extra == "otel"
@@ -41,6 +43,7 @@ Requires-Dist: faiss-cpu>=1.8; platform_system != "Windows" and extra == "produc
 Requires-Dist: qdrant-client>=1.9; extra == "production"
 Requires-Dist: psycopg[binary]>=3.1; extra == "production"
 Requires-Dist: redis>=5.0; extra == "production"
+Requires-Dist: boto3>=1.34; extra == "production"
 Requires-Dist: opentelemetry-api>=1.25; extra == "production"
 Requires-Dist: opentelemetry-sdk>=1.25; extra == "production"
 Requires-Dist: opentelemetry-exporter-otlp>=1.25; extra == "production"
@@ -543,10 +546,10 @@ Checked-in result:
 |---|---:|
 | Cluster planner | 4096 namespaces, 4 nodes, replication factor 2, node-loss availability `1.000`, zone-loss availability `1.000`, write quorum `2`. |
 | Hot cache | 2000 lookups, hit rate `0.920`, p99 lookup `0.003 ms`. |
-| Replicated runtime | 3 physical WaveMind stores, replication factor 3, write quorum 2, node-loss recall `true`, repair copied `1` missing record, tombstone repair deleted `1` stale record, p99 query-after-loss `1.57 ms`. |
-| Active-active delta sync | 2 regions, bidirectional convergence `true`, stale import suppressed after delete `true`, tombstone convergence `true`, sync `104.43 ms`. |
-| Replicated snapshot job | 3 replica files, manifest checksum validation `true`, offsite mirror validation `true`, portable archive validation `true`, archive restore `25.70 ms`, recall after restored-primary loss `true`. |
-| Structured payloads | image/audio/table/event retrieval, precision@1 `1.000`, p99 `0.58 ms`. |
+| Replicated runtime | 3 physical WaveMind stores, replication factor 3, write quorum 2, node-loss recall `true`, repair copied `1` missing record, tombstone repair deleted `1` stale record, p99 query-after-loss `1.31 ms`. |
+| Active-active delta sync | 2 regions, bidirectional convergence `true`, stale import suppressed after delete `true`, tombstone convergence `true`, sync `159.69 ms`. |
+| Replicated snapshot job | 3 replica files, manifest checksum validation `true`, offsite mirror validation `true`, portable archive validation `true`, S3-compatible upload validation `true`, archive restore `24.85 ms`. |
+| Structured payloads | image/audio/table/event retrieval, precision@1 `1.000`, p99 `0.62 ms`. |
 
 This profile validates routing, quorum-replicated runtime behavior, cache
 behavior, active-active namespace delta sync, replicated snapshot/restore, and
@@ -580,15 +583,16 @@ Maintenance workers:
 
 ```sh
 wavemind maintenance --namespace user:42 --consolidate-steps 10 --consolidate-concepts --json
-wavemind replicated-snapshot --root ./state/replicas --node node-a --node node-b --node node-c --out ./backups/replicated --offsite ./offsite/replicated --archive ./archives/replicated --keep-last 7 --json
+wavemind replicated-snapshot --root ./state/replicas --node node-a --node node-b --node node-c --out ./backups/replicated --offsite ./offsite/replicated --archive ./archives/replicated --s3 s3://my-bucket/wavemind/prod --keep-last 7 --json
 ```
 
 The first command runs one deterministic memory pass: expired-memory purge,
 optional field/concept consolidation, and index-health repair. The second
 command creates a verified replicated snapshot, mirrors it to an offsite path,
-writes a portable `.tar.gz` archive, verifies that archive, and applies
-retention. Production deployments can call these commands from cron, systemd,
-Kubernetes CronJobs, Celery, RQ, or Temporal.
+writes a portable `.tar.gz` archive, verifies that archive, can upload it to an
+S3-compatible object store, and applies retention. Production deployments can
+call these commands from cron, systemd, Kubernetes CronJobs, Celery, RQ, or
+Temporal.
 
 Hot-cache options:
 
@@ -692,6 +696,7 @@ snapshot_job = ReplicatedSnapshotWorker(memory).run_once(
     destination="./backups/replicated",
     offsite_destination="./offsite/replicated",
     archive_destination="./archives/replicated",
+    object_store_destination="s3://my-bucket/wavemind/prod",
     keep_last=7,
 )
 assert snapshot_job.ok
@@ -707,9 +712,9 @@ The replicated snapshot job writes one SQLite backup per replica plus
 `manifest.json` with SHA-256 checksums, replica metadata, quorum settings, and
 node definitions. It can mirror the snapshot to a second path for offsite
 backup, write a portable `.tar.gz` archive for object-store/offsite systems,
-verify that archive, and apply `keep_last` retention locally, offsite, and for
-archives. Restore refuses to overwrite a non-empty root unless `overwrite=True`
-is passed.
+verify that archive, upload it to any S3-compatible object store through
+`boto3`, and apply `keep_last` retention locally, offsite, and for archives.
+Restore refuses to overwrite a non-empty root unless `overwrite=True` is passed.
 
 Equivalent CLI:
 
@@ -720,6 +725,7 @@ wavemind replicated-snapshot \
   --out ./backups/replicated \
   --offsite ./offsite/replicated \
   --archive ./archives/replicated \
+  --s3 s3://my-bucket/wavemind/prod \
   --keep-last 7 \
   --json
 
@@ -728,8 +734,18 @@ wavemind replicated-restore \
   --to ./state/restored-replicas \
   --overwrite \
   --json
+
+wavemind replicated-restore \
+  --from s3://my-bucket/wavemind/prod/wavemind-replicated-20260705-120000.tar.gz \
+  --to ./state/restored-replicas \
+  --overwrite \
+  --json
 ```
 
+Install S3/R2/MinIO support with `pip install "wavemind[s3]"`. For
+S3-compatible endpoints such as Cloudflare R2 or MinIO, pass
+`--s3-endpoint-url` and optionally `--s3-region`.
+
 ## HTTP API
 
 Run the local FastAPI server:
@@ -1103,7 +1119,7 @@ Current read:
 | LongMemEval 50-query smoke | On the first 50 non-abstention LongMemEval-S questions, WaveMind reaches `evidence_recall@5 0.920`, `precision@1 0.760`, and `MRR@5 0.827`; Chroma/Qdrant static reach `0.600`, `0.260`, and `0.385`. | This is the fast regression profile for checking current changes before rerunning the full LongMemEval profile. WaveMind wins on quality; latency still needs work. |
 | ANN/index curve | At 50000 generated 128-d vectors, NumPy exact keeps `recall@10 1.000` at `6.49 ms`; quantized int8 keeps `0.934` at `24.92 ms`; Annoy is faster at `4.92 ms` but drops to `0.730` recall; Qdrant local keeps `1.000` recall at `43.49 ms`. | Current local scale boundary is clear: quantized search needs kernel work, Annoy needs tuning/FAISS, and Qdrant should be tested in service mode for a fair production comparison. |
 | Production load | At 100000 generated 128-d vectors, service-mode Qdrant reaches `recall@10 1.000`, avg `10.28 ms`, p99 `21.26 ms`. At 1M, tuned Qdrant reaches `recall@10 0.984`, avg `116.80 ms`, p99 `209.28 ms`; an EF sweep finds `recall@10 0.977`, avg `64.76 ms`, p99 `103.77 ms` at `hnsw_ef=2048` on 30 queries. | 100k is production-grade on the tested machine. 1M recall is now strong, but p99 still needs tuning before claiming a stable sub-100 ms SLO. |
-| Scale readiness | Deterministic 1M-memory simulation validates 4096 namespace placements over 4 nodes with replication factor 2, node-loss availability `1.000`, zone-loss availability `1.000`, hot-cache hit rate `0.920`, quorum-replicated runtime recall after node loss, missing-record repair, tombstone repair, active-active delta sync, checksummed replicated snapshot/restore, offsite mirror verification, portable archive verification, and structured payload precision@1 `1.000`. | This proves routing, cache, payload, replicated-runtime, namespace-delta, offsite/archive backup, and restore-drill foundations. It is not a 10M-vector latency claim; real 10M latency still needs service-backed load tests on larger hardware. |
+| Scale readiness | Deterministic 1M-memory simulation validates 4096 namespace placements over 4 nodes with replication factor 2, node-loss availability `1.000`, zone-loss availability `1.000`, hot-cache hit rate `0.920`, quorum-replicated runtime recall after node loss, missing-record repair, tombstone repair, active-active delta sync, checksummed replicated snapshot/restore, offsite mirror verification, portable archive verification, S3-compatible upload verification, and structured payload precision@1 `1.000`. | This proves routing, cache, payload, replicated-runtime, namespace-delta, offsite/archive/object-store backup, and restore-drill foundations. It is not a 10M-vector latency claim; real 10M latency still needs service-backed load tests on larger hardware. |
 | Memory competitor adapters | WaveMind reaches `precision@1 0.80`, `precision@3 1.00`, stale suppression `1.00` on the small adapter profile. Mem0, Zep, and LangGraph are listed as skipped unless their real packages/services are configured. | This prevents fake competitor claims. The adapter harness is ready; real Mem0/Zep/LangGraph results still need configured installs. |
 | LongMemEval local answer generation | With the same local Ollama `qwen2.5:1.5b`, WaveMind reaches `exact_match 0.240`, `contains_answer 0.380`, `token_f1 0.333`, and `evidence_recall@5 0.920`; Chroma and Qdrant static both reach `0.120`, `0.160`, `0.170`, and `0.600`. | This is the first checked-in end-to-end answer benchmark against Chroma/Qdrant. It is still a 50-question lightweight smoke run, not a full LongMemEval leaderboard score. |
 
@@ -1122,7 +1138,7 @@ Current read:
 | Production index profile | Docker-backed 50000-vector profile for persisted FAISS, Qdrant service, and PostgreSQL/pgvector HNSW. | implemented | FAISS / Qdrant service / pgvector | Keep service-mode candidate generation above `0.95` recall@10 and below 10 ms average query latency at 50000 vectors. |
 | Production load profile | 100k and 1M service-backed candidate-index checks with p95/p99 latency. | implemented | Qdrant service / pgvector HNSW / FAISS persisted | Keep 100k at recall@10 `1.000`; push 1M p99 below 100 ms with recall@10 >= 0.95. |
 | Qdrant 1M HNSW ef sweep | One 1M Qdrant collection queried with multiple `hnsw_ef` values. | implemented | Qdrant service | Repeat with 100+ queries and collection-level HNSW build parameters before claiming a stable 1M SLO. |
-| Scale readiness profile | Cluster placement, node/zone-loss simulation, quorum report, replicated runtime, active-active delta sync, replicated snapshot/restore with offsite and archive verification, hot-cache behavior, and structured/multimodal payload retrieval. | implemented | Mem0 / Zep / LangGraph persistent memory / GraphRAG target adapters | Keep quorum replication, namespace-delta sync, repair, offsite/archive backup, and restore drills green while adding larger service-backed 10M load tests. |
+| Scale readiness profile | Cluster placement, node/zone-loss simulation, quorum report, replicated runtime, active-active delta sync, replicated snapshot/restore with offsite, archive, and object-store verification, hot-cache behavior, and structured/multimodal payload retrieval. | implemented | Mem0 / Zep / LangGraph persistent memory / GraphRAG target adapters | Keep quorum replication, namespace-delta sync, repair, offsite/archive/object-store backup, and restore drills green while adding larger service-backed 10M load tests. |
 | Memory competitor adapter profile | Dynamic-memory scenario wired for external memory frameworks. | implemented | Mem0 / Zep / LangGraph persistent memory | Report real competitor results only when their packages/services are explicitly configured. |
 | [BEIR](https://github.com/beir-cellar/beir) | Standard zero-shot information retrieval quality. | planned | Chroma / Qdrant / FAISS | Stay within 0.02 `nDCG@10` on identical embeddings. |
 | [MTEB Retrieval](https://github.com/embeddings-benchmark/mteb) | Separates encoder quality from retrieval-store quality. | planned | Chroma / Qdrant / FAISS | Prove WaveMind does not reduce same-embedding retrieval quality. |
@@ -1298,14 +1314,15 @@ tombstone-aware, so a stale region export cannot resurrect a deleted memory.
 
 For operational recovery, `ReplicatedSnapshotWorker` creates a checksummed
 replicated snapshot, verifies an optional offsite mirror, writes a verified
-`.tar.gz` archive, and `restore_snapshot_archive()` restores it into a fresh
-replica root:
+`.tar.gz` archive, can upload that archive to S3-compatible storage, and
+`restore_snapshot_archive()` restores it into a fresh replica root:
 
 ```python
 job = ReplicatedSnapshotWorker(memory).run_once(
     destination="./backups/replicated",
     offsite_destination="./offsite/replicated",
     archive_destination="./archives/replicated",
+    object_store_destination="s3://my-bucket/wavemind/prod",
     keep_last=7,
 )
 restored, report = ReplicatedWaveMind.restore_snapshot_archive(
@@ -1315,9 +1332,10 @@ restored, report = ReplicatedWaveMind.restore_snapshot_archive(
 ```
 
 The checked-in scale-readiness profile verifies manifest checksums, verifies the
-offsite mirror, verifies the portable archive, restores three replica files from
-that archive, then disables the restored primary and confirms the memory is
-still recalled from the remaining replicas.
+offsite mirror, verifies the portable archive, verifies S3-compatible object
+upload metadata, restores three replica files from that archive, then disables
+the restored primary and confirms the memory is still recalled from the
+remaining replicas.
 
 Checked-in official LoCoMo retrieval result:
 
@@ -1707,9 +1725,10 @@ Near-term priorities:
 - Faster dynamic re-ranking through smaller candidate windows, caching, and
   background updates.
 - Better production operations: OpenTelemetry and replicated offsite snapshot
-  jobs with verified portable archives are implemented; richer latency
-  histograms, index-health metrics, alerting examples, direct object-store
-  upload adapters, and Postgres PITR runbooks are next.
+  jobs with verified portable archives and S3-compatible upload are
+  implemented; richer latency histograms, index-health metrics, alerting
+  examples, object-store retention/download drills, and Postgres PITR runbooks
+  are next.
 
 Longer-term direction:
 

{wavemind-2.2.9 → wavemind-2.3.0}/README.md

@@ -490,10 +490,10 @@ Checked-in result:
 |---|---:|
 | Cluster planner | 4096 namespaces, 4 nodes, replication factor 2, node-loss availability `1.000`, zone-loss availability `1.000`, write quorum `2`. |
 | Hot cache | 2000 lookups, hit rate `0.920`, p99 lookup `0.003 ms`. |
-| Replicated runtime | 3 physical WaveMind stores, replication factor 3, write quorum 2, node-loss recall `true`, repair copied `1` missing record, tombstone repair deleted `1` stale record, p99 query-after-loss `1.57 ms`. |
-| Active-active delta sync | 2 regions, bidirectional convergence `true`, stale import suppressed after delete `true`, tombstone convergence `true`, sync `104.43 ms`. |
-| Replicated snapshot job | 3 replica files, manifest checksum validation `true`, offsite mirror validation `true`, portable archive validation `true`, archive restore `25.70 ms`, recall after restored-primary loss `true`. |
-| Structured payloads | image/audio/table/event retrieval, precision@1 `1.000`, p99 `0.58 ms`. |
+| Replicated runtime | 3 physical WaveMind stores, replication factor 3, write quorum 2, node-loss recall `true`, repair copied `1` missing record, tombstone repair deleted `1` stale record, p99 query-after-loss `1.31 ms`. |
+| Active-active delta sync | 2 regions, bidirectional convergence `true`, stale import suppressed after delete `true`, tombstone convergence `true`, sync `159.69 ms`. |
+| Replicated snapshot job | 3 replica files, manifest checksum validation `true`, offsite mirror validation `true`, portable archive validation `true`, S3-compatible upload validation `true`, archive restore `24.85 ms`. |
+| Structured payloads | image/audio/table/event retrieval, precision@1 `1.000`, p99 `0.62 ms`. |
 
 This profile validates routing, quorum-replicated runtime behavior, cache
 behavior, active-active namespace delta sync, replicated snapshot/restore, and
@@ -527,15 +527,16 @@ Maintenance workers:
 
 ```sh
 wavemind maintenance --namespace user:42 --consolidate-steps 10 --consolidate-concepts --json
-wavemind replicated-snapshot --root ./state/replicas --node node-a --node node-b --node node-c --out ./backups/replicated --offsite ./offsite/replicated --archive ./archives/replicated --keep-last 7 --json
+wavemind replicated-snapshot --root ./state/replicas --node node-a --node node-b --node node-c --out ./backups/replicated --offsite ./offsite/replicated --archive ./archives/replicated --s3 s3://my-bucket/wavemind/prod --keep-last 7 --json
 ```
 
 The first command runs one deterministic memory pass: expired-memory purge,
 optional field/concept consolidation, and index-health repair. The second
 command creates a verified replicated snapshot, mirrors it to an offsite path,
-writes a portable `.tar.gz` archive, verifies that archive, and applies
-retention. Production deployments can call these commands from cron, systemd,
-Kubernetes CronJobs, Celery, RQ, or Temporal.
+writes a portable `.tar.gz` archive, verifies that archive, can upload it to an
+S3-compatible object store, and applies retention. Production deployments can
+call these commands from cron, systemd, Kubernetes CronJobs, Celery, RQ, or
+Temporal.
 
 Hot-cache options:
 
@@ -639,6 +640,7 @@ snapshot_job = ReplicatedSnapshotWorker(memory).run_once(
     destination="./backups/replicated",
     offsite_destination="./offsite/replicated",
     archive_destination="./archives/replicated",
+    object_store_destination="s3://my-bucket/wavemind/prod",
     keep_last=7,
 )
 assert snapshot_job.ok
@@ -654,9 +656,9 @@ The replicated snapshot job writes one SQLite backup per replica plus
 `manifest.json` with SHA-256 checksums, replica metadata, quorum settings, and
 node definitions. It can mirror the snapshot to a second path for offsite
 backup, write a portable `.tar.gz` archive for object-store/offsite systems,
-verify that archive, and apply `keep_last` retention locally, offsite, and for
-archives. Restore refuses to overwrite a non-empty root unless `overwrite=True`
-is passed.
+verify that archive, upload it to any S3-compatible object store through
+`boto3`, and apply `keep_last` retention locally, offsite, and for archives.
+Restore refuses to overwrite a non-empty root unless `overwrite=True` is passed.
 
 Equivalent CLI:
 
@@ -667,6 +669,7 @@ wavemind replicated-snapshot \
   --out ./backups/replicated \
   --offsite ./offsite/replicated \
   --archive ./archives/replicated \
+  --s3 s3://my-bucket/wavemind/prod \
   --keep-last 7 \
   --json
 
@@ -675,8 +678,18 @@ wavemind replicated-restore \
   --to ./state/restored-replicas \
   --overwrite \
   --json
+
+wavemind replicated-restore \
+  --from s3://my-bucket/wavemind/prod/wavemind-replicated-20260705-120000.tar.gz \
+  --to ./state/restored-replicas \
+  --overwrite \
+  --json
 ```
 
+Install S3/R2/MinIO support with `pip install "wavemind[s3]"`. For
+S3-compatible endpoints such as Cloudflare R2 or MinIO, pass
+`--s3-endpoint-url` and optionally `--s3-region`.
+
 ## HTTP API
 
 Run the local FastAPI server:
@@ -1050,7 +1063,7 @@ Current read:
 | LongMemEval 50-query smoke | On the first 50 non-abstention LongMemEval-S questions, WaveMind reaches `evidence_recall@5 0.920`, `precision@1 0.760`, and `MRR@5 0.827`; Chroma/Qdrant static reach `0.600`, `0.260`, and `0.385`. | This is the fast regression profile for checking current changes before rerunning the full LongMemEval profile. WaveMind wins on quality; latency still needs work. |
 | ANN/index curve | At 50000 generated 128-d vectors, NumPy exact keeps `recall@10 1.000` at `6.49 ms`; quantized int8 keeps `0.934` at `24.92 ms`; Annoy is faster at `4.92 ms` but drops to `0.730` recall; Qdrant local keeps `1.000` recall at `43.49 ms`. | Current local scale boundary is clear: quantized search needs kernel work, Annoy needs tuning/FAISS, and Qdrant should be tested in service mode for a fair production comparison. |
 | Production load | At 100000 generated 128-d vectors, service-mode Qdrant reaches `recall@10 1.000`, avg `10.28 ms`, p99 `21.26 ms`. At 1M, tuned Qdrant reaches `recall@10 0.984`, avg `116.80 ms`, p99 `209.28 ms`; an EF sweep finds `recall@10 0.977`, avg `64.76 ms`, p99 `103.77 ms` at `hnsw_ef=2048` on 30 queries. | 100k is production-grade on the tested machine. 1M recall is now strong, but p99 still needs tuning before claiming a stable sub-100 ms SLO. |
-| Scale readiness | Deterministic 1M-memory simulation validates 4096 namespace placements over 4 nodes with replication factor 2, node-loss availability `1.000`, zone-loss availability `1.000`, hot-cache hit rate `0.920`, quorum-replicated runtime recall after node loss, missing-record repair, tombstone repair, active-active delta sync, checksummed replicated snapshot/restore, offsite mirror verification, portable archive verification, and structured payload precision@1 `1.000`. | This proves routing, cache, payload, replicated-runtime, namespace-delta, offsite/archive backup, and restore-drill foundations. It is not a 10M-vector latency claim; real 10M latency still needs service-backed load tests on larger hardware. |
+| Scale readiness | Deterministic 1M-memory simulation validates 4096 namespace placements over 4 nodes with replication factor 2, node-loss availability `1.000`, zone-loss availability `1.000`, hot-cache hit rate `0.920`, quorum-replicated runtime recall after node loss, missing-record repair, tombstone repair, active-active delta sync, checksummed replicated snapshot/restore, offsite mirror verification, portable archive verification, S3-compatible upload verification, and structured payload precision@1 `1.000`. | This proves routing, cache, payload, replicated-runtime, namespace-delta, offsite/archive/object-store backup, and restore-drill foundations. It is not a 10M-vector latency claim; real 10M latency still needs service-backed load tests on larger hardware. |
 | Memory competitor adapters | WaveMind reaches `precision@1 0.80`, `precision@3 1.00`, stale suppression `1.00` on the small adapter profile. Mem0, Zep, and LangGraph are listed as skipped unless their real packages/services are configured. | This prevents fake competitor claims. The adapter harness is ready; real Mem0/Zep/LangGraph results still need configured installs. |
 | LongMemEval local answer generation | With the same local Ollama `qwen2.5:1.5b`, WaveMind reaches `exact_match 0.240`, `contains_answer 0.380`, `token_f1 0.333`, and `evidence_recall@5 0.920`; Chroma and Qdrant static both reach `0.120`, `0.160`, `0.170`, and `0.600`. | This is the first checked-in end-to-end answer benchmark against Chroma/Qdrant. It is still a 50-question lightweight smoke run, not a full LongMemEval leaderboard score. |
 
@@ -1069,7 +1082,7 @@ Current read:
 | Production index profile | Docker-backed 50000-vector profile for persisted FAISS, Qdrant service, and PostgreSQL/pgvector HNSW. | implemented | FAISS / Qdrant service / pgvector | Keep service-mode candidate generation above `0.95` recall@10 and below 10 ms average query latency at 50000 vectors. |
 | Production load profile | 100k and 1M service-backed candidate-index checks with p95/p99 latency. | implemented | Qdrant service / pgvector HNSW / FAISS persisted | Keep 100k at recall@10 `1.000`; push 1M p99 below 100 ms with recall@10 >= 0.95. |
 | Qdrant 1M HNSW ef sweep | One 1M Qdrant collection queried with multiple `hnsw_ef` values. | implemented | Qdrant service | Repeat with 100+ queries and collection-level HNSW build parameters before claiming a stable 1M SLO. |
-| Scale readiness profile | Cluster placement, node/zone-loss simulation, quorum report, replicated runtime, active-active delta sync, replicated snapshot/restore with offsite and archive verification, hot-cache behavior, and structured/multimodal payload retrieval. | implemented | Mem0 / Zep / LangGraph persistent memory / GraphRAG target adapters | Keep quorum replication, namespace-delta sync, repair, offsite/archive backup, and restore drills green while adding larger service-backed 10M load tests. |
+| Scale readiness profile | Cluster placement, node/zone-loss simulation, quorum report, replicated runtime, active-active delta sync, replicated snapshot/restore with offsite, archive, and object-store verification, hot-cache behavior, and structured/multimodal payload retrieval. | implemented | Mem0 / Zep / LangGraph persistent memory / GraphRAG target adapters | Keep quorum replication, namespace-delta sync, repair, offsite/archive/object-store backup, and restore drills green while adding larger service-backed 10M load tests. |
 | Memory competitor adapter profile | Dynamic-memory scenario wired for external memory frameworks. | implemented | Mem0 / Zep / LangGraph persistent memory | Report real competitor results only when their packages/services are explicitly configured. |
 | [BEIR](https://github.com/beir-cellar/beir) | Standard zero-shot information retrieval quality. | planned | Chroma / Qdrant / FAISS | Stay within 0.02 `nDCG@10` on identical embeddings. |
 | [MTEB Retrieval](https://github.com/embeddings-benchmark/mteb) | Separates encoder quality from retrieval-store quality. | planned | Chroma / Qdrant / FAISS | Prove WaveMind does not reduce same-embedding retrieval quality. |
@@ -1245,14 +1258,15 @@ tombstone-aware, so a stale region export cannot resurrect a deleted memory.
 
 For operational recovery, `ReplicatedSnapshotWorker` creates a checksummed
 replicated snapshot, verifies an optional offsite mirror, writes a verified
-`.tar.gz` archive, and `restore_snapshot_archive()` restores it into a fresh
-replica root:
+`.tar.gz` archive, can upload that archive to S3-compatible storage, and
+`restore_snapshot_archive()` restores it into a fresh replica root:
 
 ```python
 job = ReplicatedSnapshotWorker(memory).run_once(
     destination="./backups/replicated",
     offsite_destination="./offsite/replicated",
     archive_destination="./archives/replicated",
+    object_store_destination="s3://my-bucket/wavemind/prod",
     keep_last=7,
 )
 restored, report = ReplicatedWaveMind.restore_snapshot_archive(
@@ -1262,9 +1276,10 @@ restored, report = ReplicatedWaveMind.restore_snapshot_archive(
 ```
 
 The checked-in scale-readiness profile verifies manifest checksums, verifies the
-offsite mirror, verifies the portable archive, restores three replica files from
-that archive, then disables the restored primary and confirms the memory is
-still recalled from the remaining replicas.
+offsite mirror, verifies the portable archive, verifies S3-compatible object
+upload metadata, restores three replica files from that archive, then disables
+the restored primary and confirms the memory is still recalled from the
+remaining replicas.
 
 Checked-in official LoCoMo retrieval result:
 
@@ -1654,9 +1669,10 @@ Near-term priorities:
 - Faster dynamic re-ranking through smaller candidate windows, caching, and
   background updates.
 - Better production operations: OpenTelemetry and replicated offsite snapshot
-  jobs with verified portable archives are implemented; richer latency
-  histograms, index-health metrics, alerting examples, direct object-store
-  upload adapters, and Postgres PITR runbooks are next.
+  jobs with verified portable archives and S3-compatible upload are
+  implemented; richer latency histograms, index-health metrics, alerting
+  examples, object-store retention/download drills, and Postgres PITR runbooks
+  are next.
 
 Longer-term direction:
 

{wavemind-2.2.9 → wavemind-2.3.0}/benchmarks/BENCHMARK_LEADERBOARD.md

@@ -21,7 +21,7 @@ This is a compact reader-facing view of checked-in benchmark results. It is not
 | Production load profile 100k | production-scale | Recall@k | WaveMind pgvector: 0.736 / 17.8 ms | Qdrant service: 1 / 10.3 ms | Baseline leads on quality |
 | Production load profile 1M | production-scale | Recall@k | - | Qdrant service: 0.984 / 116.8 ms | No WaveMind result |
 | Qdrant 1M HNSW ef sweep | production-scale | Recall@k | - | hnsw_ef=2048: 0.977 / 64.8 ms | No WaveMind result |
-| Scale readiness profile | production-scale | precision@1 | WaveMind structured payloads: 1 / 0.461 ms | - | WaveMind-only check |
+| Scale readiness profile | production-scale | precision@1 | WaveMind structured payloads: 1 / 0.41 ms | - | WaveMind-only check |
 | Memory competitor adapter profile | agent-memory | precision@1 | WaveMind: 0.8 / 0.554 ms | - | WaveMind-only check |
 | [LongMemEval answer generation](https://github.com/xiaowu0162/LongMemEval) | long-term-agent-memory | token F1 | WaveMind + qwen2.5:1.5b: 0.333 / - | Chroma static + qwen2.5:1.5b: 0.17 / - | WaveMind leads on quality |
 

{wavemind-2.2.9 → wavemind-2.3.0}/benchmarks/BENCHMARK_REPORT.md

@@ -24,7 +24,7 @@ Planned rows are not claimed wins. They are the public proof path WaveMind must
 | Production load profile 100k | production-scale | implemented | Qdrant service: Recall@k 1.00, avg latency 10.3, p95 latency 19.0, p99 latency ms 21.3, build ms 27439.3<br>WaveMind pgvector: Recall@k 0.74, avg latency 17.8, p95 latency 23.5, build ms 455703.7<br>WaveMind faiss-persisted: skipped - Set WAVEMIND_FAISS_PATH to use the persisted FAISS backend | Tune pgvector HNSW build/search parameters and add persisted FAISS from the Linux benchmark container. |
 | Production load profile 1M | production-scale | implemented | Qdrant service: Recall@k 0.98, avg latency 116.8, p95 latency 153.8, p99 latency ms 209.3, build ms 450674.6 | Tune Qdrant indexing/search params further, then add FAISS IVF/HNSW and pgvector 1M profiles on a larger disk. |
 | Qdrant 1M HNSW ef sweep | production-scale | implemented | hnsw_ef=512: Recall@k 0.75, avg latency 47.2, p95 latency 68.5, p99 latency ms 68.5, max latency ms 68.5<br>hnsw_ef=768: Recall@k 0.85, avg latency 44.0, p95 latency 69.1, p99 latency ms 69.8, max latency ms 69.8<br>hnsw_ef=1024: Recall@k 0.88, avg latency 62.9, p95 latency 81.1, p99 latency ms 85.5, max latency ms 85.5<br>hnsw_ef=1536: Recall@k 0.94, avg latency 65.6, p95 latency 111.2, p99 latency ms 119.7, max latency ms 119.7<br>hnsw_ef=2048: Recall@k 0.98, avg latency 64.8, p95 latency 91.2, p99 latency ms 103.8, max latency ms 103.8 | Repeat with 100+ queries and collection-level HNSW build parameters before claiming a stable production SLO. |
-| Scale readiness profile | production-scale | implemented | WaveMind cluster planner: simulated memories 1000000, namespaces 4096, nodes 4, replication factor 2, node loss min availability 1.00, zone loss min availability 1.00, read quorum 1, write quorum 2, placement ms 61.5<br>WaveMind hot cache: queries 2000, capacity 512, hit rate 0.92, evictions 0, p99 lookup ms 0.00<br>WaveMind replicated runtime: nodes 3, replication factor 3, write quorum 2, read quorum 1, recalled after node loss True, repair copied records 1, tombstone repair deleted records 1, p99 query after loss ms 1.57<br>WaveMind active-active delta sync: regions 2, replication factor per region 3, records imported 6, converged after bidirectional sync True, suppressed stale import after delete True, tombstone converged True, sync ms 104.4<br>WaveMind replicated snapshot: nodes 3, manifest healthy True, offsite verified True, archive verified True, restored files 3, recalled after restore node loss True, snapshot ms 109.2, restore ms 25.7<br>WaveMind structured payloads: queries 4, precision@1 1.00, avg latency 0.46, p99 latency ms 0.58 | Move from local replicated runtime to service-backed replicated runs, object-store snapshot adapters, and larger 10M candidate-index load tests. |
+| Scale readiness profile | production-scale | implemented | WaveMind cluster planner: simulated memories 1000000, namespaces 4096, nodes 4, replication factor 2, node loss min availability 1.00, zone loss min availability 1.00, read quorum 1, write quorum 2, placement ms 62.6<br>WaveMind hot cache: queries 2000, capacity 512, hit rate 0.92, evictions 0, p99 lookup ms 0.00<br>WaveMind replicated runtime: nodes 3, replication factor 3, write quorum 2, read quorum 1, recalled after node loss True, repair copied records 1, tombstone repair deleted records 1, p99 query after loss ms 1.31<br>WaveMind active-active delta sync: regions 2, replication factor per region 3, records imported 6, converged after bidirectional sync True, suppressed stale import after delete True, tombstone converged True, sync ms 159.7<br>WaveMind replicated snapshot: nodes 3, manifest healthy True, offsite verified True, archive verified True, object store verified True, restored files 3, recalled after restore node loss True, snapshot ms 186.2, restore ms 24.9<br>WaveMind structured payloads: queries 4, precision@1 1.00, avg latency 0.41, p99 latency ms 0.62 | Move from local replicated runtime to service-backed replicated runs, object-store retention/download drills, and larger 10M candidate-index load tests. |
 | Memory competitor adapter profile | agent-memory | implemented | WaveMind: precision@1 0.80, precision@3 1.00, stale suppression 1.00, avg latency 0.55, p95 latency 0.83<br>Mem0: skipped - Install Mem0 to run this adapter profile: pip install "mem0ai"<br>Zep: skipped - Install the Zep client package and set ZEP_API_KEY or ZEP_API_URL.<br>LangGraph persistent memory: skipped - Install LangGraph to run this adapter profile: pip install "langgraph" | Add documented setup commands for each competitor adapter and store checked-in results only when those real adapters run. |
 | [LongMemEval answer generation](https://github.com/xiaowu0162/LongMemEval) | long-term-agent-memory | implemented | extractive smoke: queries 20, evidence recall@k 1.00, exact match 0.00, contains answer 0.05, token f1 0.02, avg retrieval ms 3.79, avg generation ms 0.77<br>WaveMind + qwen2.5:0.5b: queries 50, evidence recall@k 0.92, exact match 0.12, contains answer 0.18, token f1 0.18, avg retrieval ms 2.98, avg generation ms 1428.2<br>Chroma static + qwen2.5:0.5b: queries 50, evidence recall@k 0.60, exact match 0.10, contains answer 0.12, token f1 0.13, avg retrieval ms 4.10, avg generation ms 1234.7<br>Qdrant static + qwen2.5:0.5b: queries 50, evidence recall@k 0.60, exact match 0.10, contains answer 0.12, token f1 0.13, avg retrieval ms 63.8, avg generation ms 893.5<br>WaveMind + qwen2.5:1.5b: queries 50, evidence recall@k 0.92, exact match 0.24, contains answer 0.38, token f1 0.33, avg retrieval ms 2.00, avg generation ms 2153.0<br>Chroma static + qwen2.5:1.5b: queries 50, evidence recall@k 0.60, exact match 0.12, contains answer 0.16, token f1 0.17, avg retrieval ms 7.05, avg generation ms 2082.4<br>Qdrant static + qwen2.5:1.5b: queries 50, evidence recall@k 0.60, exact match 0.12, contains answer 0.16, token f1 0.17, avg retrieval ms 100.2, avg generation ms 758.1 | Run all 470 non-abstention questions with a stronger local/API model and add faithfulness/abstention scoring. |
 

{wavemind-2.2.9 → wavemind-2.3.0}/benchmarks/benchmark_matrix_results.json

@@ -741,7 +741,7 @@
       "category": "production-scale",
       "status": "implemented",
       "source": "benchmarks/scale_readiness_benchmark.py",
-      "dataset": "Deterministic 1M-memory simulation for namespace placement, quorum runtime, active-active delta sync, replicated snapshot/offsite/archive restore, hot-cache, and structured-payload retrieval checks.",
+      "dataset": "Deterministic 1M-memory simulation for namespace placement, quorum runtime, active-active delta sync, replicated snapshot/offsite/archive restore, S3-compatible object-store upload verification, hot-cache, and structured-payload retrieval checks.",
       "competitors": [
         "Mem0",
         "Zep",
@@ -764,14 +764,14 @@
           "zone_loss_min_availability": 1.0,
           "read_quorum": 1,
           "write_quorum": 2,
-          "placement_ms": 61.49380002170801
+          "placement_ms": 62.552399991545826
         },
         "WaveMind hot cache": {
           "queries": 2000,
           "capacity": 512,
           "hit_rate": 0.92,
           "evictions": 0,
-          "p99_lookup_ms": 0.002900022082030773
+          "p99_lookup_ms": 0.0026999623514711857
         },
         "WaveMind replicated runtime": {
           "nodes": 3,
@@ -781,7 +781,7 @@
           "recalled_after_node_loss": true,
           "repair_copied_records": 1,
           "tombstone_repair_deleted_records": 1,
-          "p99_query_after_loss_ms": 1.573899993672967
+          "p99_query_after_loss_ms": 1.3142999960109591
         },
         "WaveMind active-active delta sync": {
           "regions": 2,
@@ -790,27 +790,28 @@
           "converged_after_bidirectional_sync": true,
           "suppressed_stale_import_after_delete": true,
           "tombstone_converged": true,
-          "sync_ms": 104.4289999990724
+          "sync_ms": 159.69160001259297
         },
         "WaveMind replicated snapshot": {
           "nodes": 3,
           "manifest_healthy": true,
           "offsite_verified": true,
           "archive_verified": true,
+          "object_store_verified": true,
           "restored_files": 3,
           "recalled_after_restore_node_loss": true,
-          "snapshot_ms": 109.23659999389201,
-          "restore_ms": 25.69669997319579
+          "snapshot_ms": 186.24169996473938,
+          "restore_ms": 24.853000009898096
         },
         "WaveMind structured payloads": {
           "queries": 4,
           "precision_at_1": 1.0,
-          "avg_latency_ms": 0.4606249858625233,
-          "p99_latency_ms": 0.5845999694429338
+          "avg_latency_ms": 0.4099500220036134,
+          "p99_latency_ms": 0.6203000084497035
         }
       },
-      "target": "Prove the production foundation before heavier 100k, 1M, and 10M vector load tests: deterministic placement, survivable replicas, active-active sync, offsite/archive-verified restore drills, hot-cache behavior, and structured payload recall.",
-      "next_step": "Move from local replicated runtime to service-backed replicated runs, object-store snapshot adapters, and larger 10M candidate-index load tests."
+      "target": "Prove the production foundation before heavier 100k, 1M, and 10M vector load tests: deterministic placement, survivable replicas, active-active sync, offsite/archive/object-store-verified restore drills, hot-cache behavior, and structured payload recall.",
+      "next_step": "Move from local replicated runtime to service-backed replicated runs, object-store retention/download drills, and larger 10M candidate-index load tests."
     },
     {
       "id": "memory_competitor_adapter_profile",

{wavemind-2.2.9 → wavemind-2.3.0}/benchmarks/benchmark_registry.py

@@ -702,7 +702,7 @@ def _implemented_entries(root: Path) -> list[dict[str, Any]]:
             "category": "production-scale",
             "status": "implemented",
             "source": "benchmarks/scale_readiness_benchmark.py",
-            "dataset": "Deterministic 1M-memory simulation for namespace placement, quorum runtime, active-active delta sync, replicated snapshot/offsite/archive restore, hot-cache, and structured-payload retrieval checks.",
+            "dataset": "Deterministic 1M-memory simulation for namespace placement, quorum runtime, active-active delta sync, replicated snapshot/offsite/archive restore, S3-compatible object-store upload verification, hot-cache, and structured-payload retrieval checks.",
             "competitors": ["Mem0", "Zep", "LangGraph persistent memory", "GraphRAG"],
             "metrics": [
                 "node_loss_min_availability",
@@ -767,6 +767,7 @@ def _implemented_entries(root: Path) -> list[dict[str, Any]]:
                         "manifest_healthy",
                         "offsite_verified",
                         "archive_verified",
+                        "object_store_verified",
                         "restored_files",
                         "recalled_after_restore_node_loss",
                         "snapshot_ms",
@@ -783,8 +784,8 @@ def _implemented_entries(root: Path) -> list[dict[str, Any]]:
                     ),
                 ),
             },
-            "target": "Prove the production foundation before heavier 100k, 1M, and 10M vector load tests: deterministic placement, survivable replicas, active-active sync, offsite/archive-verified restore drills, hot-cache behavior, and structured payload recall.",
-            "next_step": "Move from local replicated runtime to service-backed replicated runs, object-store snapshot adapters, and larger 10M candidate-index load tests.",
+            "target": "Prove the production foundation before heavier 100k, 1M, and 10M vector load tests: deterministic placement, survivable replicas, active-active sync, offsite/archive/object-store-verified restore drills, hot-cache behavior, and structured payload recall.",
+            "next_step": "Move from local replicated runtime to service-backed replicated runs, object-store retention/download drills, and larger 10M candidate-index load tests.",
         },
         {
             "id": "memory_competitor_adapter_profile",

{wavemind-2.2.9 → wavemind-2.3.0}/benchmarks/scale_readiness_benchmark.py

@@ -20,6 +20,7 @@ from wavemind import (
     QueryResult,
     ReplicatedWaveMind,
     ReplicatedSnapshotWorker,
+    S3SnapshotStore,
     WaveMind,
     audio_payload,
     build_cluster_plan,
@@ -31,6 +32,26 @@ from wavemind import (
 )
 
 
+class InMemoryS3Client:
+    def __init__(self):
+        self.objects: dict[tuple[str, str], dict[str, object]] = {}
+
+    def upload_file(self, filename: str, bucket: str, key: str, ExtraArgs=None):
+        self.objects[(bucket, key)] = {
+            "Body": Path(filename).read_bytes(),
+            "Metadata": dict((ExtraArgs or {}).get("Metadata") or {}),
+        }
+
+    def head_object(self, Bucket: str, Key: str) -> dict[str, object]:
+        payload = self.objects[(Bucket, Key)]
+        body = payload["Body"]
+        return {
+            "ContentLength": len(body),
+            "Metadata": dict(payload["Metadata"]),
+            "ETag": '"benchmark-etag"',
+        }
+
+
 def percentile(values: list[float], pct: float) -> float:
     if not values:
         return 0.0
@@ -336,10 +357,16 @@ def run_replicated_snapshot_profile() -> dict[str, object]:
                 namespace=namespace,
             )
             snapshot_started = time.perf_counter()
+            object_store = S3SnapshotStore.from_uri(
+                "s3://wavemind-benchmark/replicated",
+                client=InMemoryS3Client(),
+            )
             snapshot_job = ReplicatedSnapshotWorker(memory).run_once(
                 destination=root / "snapshots",
                 offsite_destination=root / "offsite",
                 archive_destination=root / "archives",
+                object_store_destination="s3://wavemind-benchmark/replicated",
+                object_store=object_store,
                 keep_last=2,
             )
             snapshot_ms = (time.perf_counter() - snapshot_started) * 1000.0
@@ -366,6 +393,10 @@ def run_replicated_snapshot_profile() -> dict[str, object]:
                 "manifest_healthy": health["healthy"],
                 "offsite_verified": snapshot_job.offsite_verified,
                 "archive_verified": snapshot_job.archive_verified,
+                "object_store_verified": bool(
+                    snapshot_job.object_store_upload
+                    and snapshot_job.object_store_upload.verified
+                ),
                 "total_bytes": snapshot_job.total_bytes,
                 "snapshot_ms": snapshot_ms,
                 "restore_ms": restore_ms,
@@ -480,8 +511,9 @@ def run_benchmark(
                 "Deterministic scale-readiness profile for cluster placement, "
                 "node/zone loss simulation, quorum-replicated runtime behavior, "
                 "active-active delta sync, replicated snapshot/offsite/archive "
-                "restore, hot-cache behavior, and structured payload retrieval. This is not a "
-                "10M-vector database load test."
+                "restore, S3-compatible object-store upload verification, hot-cache "
+                "behavior, and structured payload retrieval. This is not a 10M-vector "
+                "database load test."
             ),
         },
         "results": results,
@@ -529,6 +561,7 @@ def main() -> int:
             print(f"| replicated snapshot | manifest_healthy | {result['manifest_healthy']} |")
             print(f"| replicated snapshot | offsite_verified | {result['offsite_verified']} |")
             print(f"| replicated snapshot | archive_verified | {result['archive_verified']} |")
+            print(f"| replicated snapshot | object_store_verified | {result['object_store_verified']} |")
             print(f"| replicated snapshot | recalled_after_restore_node_loss | {result['recalled_after_restore_node_loss']} |")
         else:
             print(f"| structured payloads | precision@1 | {result['precision_at_1']:.3f} |")

{wavemind-2.2.9 → wavemind-2.3.0}/benchmarks/scale_readiness_results.json

@@ -5,7 +5,7 @@
     "namespace_count": 4096,
     "node_count": 4,
     "replication_factor": 2,
-    "description": "Deterministic scale-readiness profile for cluster placement, node/zone loss simulation, quorum-replicated runtime behavior, active-active delta sync, replicated snapshot/offsite/archive restore, hot-cache behavior, and structured payload retrieval. This is not a 10M-vector database load test."
+    "description": "Deterministic scale-readiness profile for cluster placement, node/zone loss simulation, quorum-replicated runtime behavior, active-active delta sync, replicated snapshot/offsite/archive restore, S3-compatible object-store upload verification, hot-cache behavior, and structured payload retrieval. This is not a 10M-vector database load test."
   },
   "results": [
     {
@@ -14,7 +14,7 @@
       "namespaces": 4096,
       "nodes": 4,
       "replication_factor": 2,
-      "placement_ms": 61.49380002170801,
+      "placement_ms": 62.552399991545826,
       "max_replica_load": 2413,
       "min_replica_load": 1728,
       "replica_load_stdev": 316.54462560593254,
@@ -32,8 +32,8 @@
       "capacity": 512,
       "hit_rate": 0.92,
       "evictions": 0,
-      "avg_lookup_ms": 0.0013803999754600227,
-      "p99_lookup_ms": 0.002900022082030773
+      "avg_lookup_ms": 0.0013917501782998443,
+      "p99_lookup_ms": 0.0026999623514711857
     },
     {
       "engine": "WaveMind replicated runtime",
@@ -47,8 +47,8 @@
       "tombstone_suppressed_before_repair": true,
       "tombstone_suppressed_after_repair": true,
       "tombstone_repair_deleted_records": 1,
-      "avg_query_after_loss_ms": 1.573899993672967,
-      "p99_query_after_loss_ms": 1.573899993672967
+      "avg_query_after_loss_ms": 1.3142999960109591,
+      "p99_query_after_loss_ms": 1.3142999960109591
     },
     {
       "engine": "WaveMind active-active delta sync",
@@ -56,7 +56,7 @@
       "replication_factor_per_region": 3,
       "records_imported": 6,
       "converged_after_bidirectional_sync": true,
-      "sync_ms": 104.4289999990724,
+      "sync_ms": 159.69160001259297,
       "suppressed_stale_import_after_delete": true,
       "tombstone_deleted_records": 3,
       "tombstone_converged": true
@@ -67,9 +67,10 @@
       "manifest_healthy": true,
       "offsite_verified": true,
       "archive_verified": true,
+      "object_store_verified": true,
       "total_bytes": 110592,
-      "snapshot_ms": 109.23659999389201,
-      "restore_ms": 25.69669997319579,
+      "snapshot_ms": 186.24169996473938,
+      "restore_ms": 24.853000009898096,
       "restored_files": 3,
       "recalled_after_restore_node_loss": true
     },
@@ -83,8 +84,8 @@
       ],
       "queries": 4,
       "precision_at_1": 1.0,
-      "avg_latency_ms": 0.4606249858625233,
-      "p99_latency_ms": 0.5845999694429338
+      "avg_latency_ms": 0.4099500220036134,
+      "p99_latency_ms": 0.6203000084497035
     }
   ]
 }

{wavemind-2.2.9 → wavemind-2.3.0}/docker-compose.yml

@@ -4,7 +4,7 @@ services:
       context: .
       args:
         INSTALL_OPTIONAL: "false"
-    image: wavemind:2.2.9
+    image: wavemind:2.3.0
     restart: unless-stopped
     environment:
       WAVEMIND_DB: /data/wavemind.sqlite3