fleetmesh-node 0.2.0__py3-none-any.whl

node/protocols/nostr/blobstore.py

@@ -0,0 +1,383 @@
+"""Content-Addressed Storage (CAS) for blobs and IPFS CIDv1 interoperability.
+
+Spec reference: spec.html section "Content Addressing & CIDs" (lines 3470-3476) and "Blobs" (lines 3768-3776).
+- Blossom blobs are SHA-256 addressed.
+- Maps directly to IPFS Content Identifiers (CIDs):
+    multibase 'b' (RFC 4648 Base32 lowercase), CIDv1 (0x01), raw binary codec (0x55), multihash sha2-256 (0x12, 0x20).
+- Standard CAS on-disk sharded directory layout: `blobs/data/<sha256[:2]>/<sha256>`
+  (fanout depth 1). The depth is configurable (0..8 two-hex-char levels) so an
+  operator may point the store at a tree another CAS already lays out the same
+  way (magpie: `<data>/objects/<ab>/<cd>/<sha>`, depth 2) - see docs/sidecars.md.
+  The depth a directory was created with is recorded in `blobs.sqlite` and a
+  store opened with a different depth refuses rather than silently 404 every blob.
+- SQLite metadata index in `blobs.sqlite` tracking size, content-type, uploader attribution, and upload timestamps.
+"""
+from __future__ import annotations
+
+import hashlib
+import os
+import re
+import tempfile
+import time
+from typing import Optional, TypedDict
+
+from fleetmesh_core.identity.cid import cid_to_sha256, sha256_to_cid
+from node.compartment.schema import Column, Index, Migration, Table
+from node.storage import SqliteStore
+
+HEX64_RE = re.compile(r'^[0-9a-f]{64}$')
+
+__all__ = ['BlobDescriptor', 'BlobMetadataStore', 'BlobStore', 'cid_to_sha256', 'sha256_to_cid']
+
+
+class BlobDescriptor(TypedDict, total=False):
+    """Metadata descriptor for a stored or indexed blob."""
+    sha256: str
+    size: int
+    type: str
+    uploaded: int
+    uploaded_by: Optional[str]
+    cid: str
+    pinned: bool
+    accessed_at: Optional[int]
+
+
+class BlobMetadataStore(SqliteStore):
+    """Thread-safe SQLite metadata index for Content-Addressed Storage."""
+
+    SCHEMA = (
+        Table('blobs', (
+            Column('sha256', 'TEXT', primary_key=True),
+            Column('size', 'INTEGER', not_null=True),
+            Column('type', 'TEXT', not_null=True),
+            Column('uploaded_by', 'TEXT'),
+            Column('created_at', 'INTEGER', not_null=True),
+            Column('accessed_at', 'INTEGER'),
+            Column('pinned', 'INTEGER', default=0),
+        ), indices=(
+            Index('idx_blobs_uploaded', ('uploaded_by',)),
+        )),
+        Table('meta', (
+            Column('key', 'TEXT', primary_key=True),
+            Column('value', 'TEXT', not_null=True),
+        )),
+    )
+    # A blobs table from before the LRU existed lacks both columns; the index
+    # over them can only be built once they are there, so it is the last step.
+    MIGRATIONS = (
+        Migration(1, up='ALTER TABLE blobs ADD COLUMN accessed_at INTEGER', name='blobs_accessed_at'),
+        Migration(2, up='ALTER TABLE blobs ADD COLUMN pinned INTEGER DEFAULT 0', name='blobs_pinned'),
+        Migration(3, up='CREATE INDEX IF NOT EXISTS idx_blobs_lru ON blobs(pinned, accessed_at)', name='idx_blobs_lru'),
+    )
+
+    def meta_get(self, key: str) -> Optional[str]:
+        row = self._one('SELECT value FROM meta WHERE key = ?', (key,))
+        return None if row is None else str(row[0])
+
+    def meta_set_if_absent(self, key: str, value: str) -> str:
+        """Record ``value`` under ``key`` unless one is already there; return the stored one."""
+        self.execute('INSERT OR IGNORE INTO meta(key, value) VALUES (?, ?)', (key, value))
+        self.commit()
+        got = self.meta_get(key)
+        return value if got is None else got
+
+
+class LayoutMismatch(RuntimeError):
+    """A blob directory was created with a different fanout depth than it is being opened with."""
+
+
+MAX_FANOUT = 8
+DEFAULT_FANOUT = 1
+
+
+class BlobStore:
+    """Content-Addressed Storage engine for SHA-256 blobs and artifacts."""
+
+    def __init__(
+        self,
+        directory: str,
+        max_bytes: Optional[int] = None,
+        max_storage_bytes: Optional[int] = None,
+        fanout: int = DEFAULT_FANOUT,
+        data_dir: Optional[str] = None,
+    ) -> None:
+        """``directory`` holds ``blobs.sqlite``; blobs live under ``data_dir``
+        (default ``<directory>/data``) at ``fanout`` two-hex-char levels.
+
+        ``fanout``/``data_dir`` exist so a node may share a tree with another
+        CAS that uses the same sha256 layout. Shared disk is not a shared
+        catalogue - each index knows only what it wrote - and the other writer's
+        file modes/ownership must let this process read and unlink.
+        """
+        if not isinstance(fanout, int) or isinstance(fanout, bool) or not 0 <= fanout <= MAX_FANOUT:
+            raise ValueError(f'blob fanout must be an integer 0..{MAX_FANOUT}, got {fanout!r}')
+        self.dir: str = os.path.realpath(directory)
+        self.data_dir: str = os.path.realpath(data_dir) if data_dir else os.path.join(self.dir, 'data')
+        self.db_path: str = os.path.join(self.dir, 'blobs.sqlite')
+        self.max_bytes: Optional[int] = max_bytes
+        self.max_storage_bytes: Optional[int] = max_storage_bytes
+        self.fanout: int = fanout
+
+        os.makedirs(self.dir, exist_ok=True)
+        os.makedirs(self.data_dir, exist_ok=True)
+        self.db: BlobMetadataStore = BlobMetadataStore(self.db_path)
+        recorded = self.db.meta_set_if_absent('fanout', str(fanout))
+        if recorded != str(fanout):
+            self.db.close()
+            raise LayoutMismatch(
+                f'{self.db.safe_path} was created with fanout {recorded}; opening with {fanout} '
+                f'would hide every stored blob. Keep the recorded depth or point at a new directory.')
+        recorded_dir = self.db.meta_set_if_absent('data_dir', self.data_dir)
+        if recorded_dir != self.data_dir:
+            self.db.close()
+            raise LayoutMismatch(
+                f'{self.db.safe_path} indexes blobs under {recorded_dir}, not {self.data_dir}. '
+                f'Keep the recorded data_dir or point at a new directory.')
+
+    def _blob_path(self, sha256_hex: str) -> str:
+        h = sha256_hex.lower()
+        levels = [h[2 * i:2 * i + 2] for i in range(self.fanout)]
+        return os.path.join(self.data_dir, *levels, h)
+
+    def put(
+        self,
+        data: bytes,
+        content_type: str = 'application/octet-stream',
+        uploaded_by: Optional[str] = None,
+        now: Optional[int] = None,
+    ) -> BlobDescriptor:
+        """Store a blob by content hash. Returns normative descriptor."""
+        if not isinstance(data, (bytes, bytearray)):
+            raise TypeError("blob data must be bytes")
+
+        size = len(data)
+        if self.max_bytes is not None and size > self.max_bytes:
+            raise ValueError(f"blob size {size} exceeds maximum limit {self.max_bytes}")
+
+        h = hashlib.sha256(data).hexdigest().lower()
+        now_ts = int(now if now is not None else time.time())
+
+        # Enforce quota watermark with automatic LRU eviction
+        if self.max_storage_bytes is not None and not self.has(h):
+            cur_total = self.total_bytes()
+            if cur_total + size > self.max_storage_bytes:
+                needed = (cur_total + size) - self.max_storage_bytes
+                self.evict_lru(needed_bytes=needed)
+                if self.total_bytes() + size > self.max_storage_bytes:
+                    raise ValueError(
+                        f"CAS quota exceeded: cannot store {size} bytes (storage cap {self.max_storage_bytes} bytes)"
+                    )
+
+        dest = self._blob_path(h)
+        os.makedirs(os.path.dirname(dest), exist_ok=True)
+
+        # Write atomically via tempfile in same filesystem directory
+        if not os.path.exists(dest):
+            with tempfile.NamedTemporaryFile(dir=os.path.dirname(dest), delete=False) as tf:
+                tf.write(data)
+                tmp_name = tf.name
+            os.replace(tmp_name, dest)
+
+        # Record metadata preserving pinned state if existing
+        self.db.execute('''
+            INSERT INTO blobs (sha256, size, type, uploaded_by, created_at, accessed_at, pinned)
+            VALUES (?, ?, ?, ?, ?, ?, COALESCE((SELECT pinned FROM blobs WHERE sha256 = ?), 0))
+            ON CONFLICT(sha256) DO UPDATE SET
+                size=excluded.size,
+                type=excluded.type,
+                accessed_at=excluded.accessed_at
+        ''', (h, size, content_type, uploaded_by, now_ts, now_ts, h))
+        self.db.commit()
+
+        return {
+            'sha256': h,
+            'size': size,
+            'type': content_type,
+            'uploaded': now_ts,
+            'uploaded_by': uploaded_by,
+            'cid': sha256_to_cid(h),
+        }
+
+    def get(self, sha256_hex: str) -> Optional[bytes]:
+        """Read full blob bytes, verifying SHA-256 integrity."""
+        h = sha256_hex.lower()
+        if not HEX64_RE.match(h):
+            return None
+
+        path = self._blob_path(h)
+        if not os.path.exists(path):
+            return None
+
+        try:
+            with open(path, 'rb') as f:
+                data = f.read()
+            # Verify integrity
+            if hashlib.sha256(data).hexdigest().lower() != h:
+                os.unlink(path)
+                return None
+            try:
+                self.db.execute('UPDATE blobs SET accessed_at = ? WHERE sha256 = ?', (int(time.time()), h))
+                self.db.commit()
+            except Exception:
+                pass
+            return data
+        except OSError:
+            return None
+
+    def has(self, sha256_hex: str) -> bool:
+        """Check if blob exists on disk and is indexed."""
+        h = sha256_hex.lower()
+        if not HEX64_RE.match(h):
+            return False
+        return os.path.exists(self._blob_path(h))
+
+    def stat(self, sha256_hex: str) -> Optional[BlobDescriptor]:
+        """Retrieve metadata descriptor without loading blob content."""
+        h = sha256_hex.lower()
+        if not HEX64_RE.match(h):
+            return None
+
+        row = self.db._one('''
+            SELECT size, type, uploaded_by, created_at, pinned, accessed_at FROM blobs WHERE sha256 = ?
+        ''', (h,))
+
+        if not row:
+            if not os.path.exists(self._blob_path(h)):
+                return None
+            try:
+                sz = os.path.getsize(self._blob_path(h))
+                return {
+                    'sha256': h,
+                    'size': sz,
+                    'type': 'application/octet-stream',
+                    'uploaded': int(os.path.getmtime(self._blob_path(h))),
+                    'uploaded_by': None,
+                    'cid': sha256_to_cid(h),
+                    'pinned': False,
+                }
+            except OSError:
+                return None
+
+        size, ctype, uploaded_by, created_at, pinned, accessed_at = row
+        return {
+            'sha256': h,
+            'size': size,
+            'type': ctype,
+            'uploaded': created_at,
+            'uploaded_by': uploaded_by,
+            'cid': sha256_to_cid(h),
+            'pinned': bool(pinned),
+            'accessed_at': accessed_at,
+        }
+
+    def delete(self, sha256_hex: str) -> bool:
+        """Delete blob from disk and metadata index.
+
+        The digest is validated before it touches the filesystem: get/has
+        already refuse anything but 64 hex chars, and delete is the one
+        path here that unlinks, so '../..' must never reach _blob_path.
+        """
+        h = sha256_hex.lower()
+        if not HEX64_RE.match(h):
+            return False
+        path = self._blob_path(h)
+        deleted = False
+        if os.path.exists(path):
+            try:
+                os.unlink(path)
+                deleted = True
+            except OSError:
+                pass
+
+        cur = self.db.execute('DELETE FROM blobs WHERE sha256 = ?', (h,))
+        self.db.commit()
+        if cur.rowcount > 0:
+            deleted = True
+
+        return deleted
+
+    def list_by(
+        self,
+        uploaded_by: Optional[str] = None,
+        limit: int = 100,
+        offset: int = 0,
+    ) -> list[BlobDescriptor]:
+        """List stored blobs matching uploader pubkey."""
+        if uploaded_by:
+            rows = self.db._all('''
+                SELECT sha256, size, type, uploaded_by, created_at
+                FROM blobs WHERE uploaded_by = ?
+                ORDER BY created_at DESC LIMIT ? OFFSET ?
+            ''', (uploaded_by, limit, offset))
+        else:
+            rows = self.db._all('''
+                SELECT sha256, size, type, uploaded_by, created_at
+                FROM blobs ORDER BY created_at DESC LIMIT ? OFFSET ?
+            ''', (limit, offset))
+
+        out: list[BlobDescriptor] = []
+        for h, sz, ctype, uploader, ts in rows:
+            out.append({
+                'sha256': h,
+                'size': sz,
+                'type': ctype,
+                'uploaded': ts,
+                'uploaded_by': uploader,
+                'cid': sha256_to_cid(h),
+            })
+        return out
+
+    def total_bytes(self) -> int:
+        """Total size in bytes of all indexed blobs."""
+        row = self.db._one('SELECT SUM(size) FROM blobs')
+        return int((row and row[0]) or 0)
+
+    def count(self) -> int:
+        """Number of indexed blobs."""
+        row = self.db._one('SELECT COUNT(*) FROM blobs')
+        return int((row and row[0]) or 0)
+
+    def pin(self, sha256_hex: str) -> bool:
+        """Pin a blob to prevent LRU eviction."""
+        h = sha256_hex.lower()
+        if not HEX64_RE.match(h) or not self.has(h):
+            return False
+        cur = self.db.execute('UPDATE blobs SET pinned = 1 WHERE sha256 = ?', (h,))
+        self.db.commit()
+        return cur.rowcount > 0
+
+    def unpin(self, sha256_hex: str) -> bool:
+        """Unpin a blob, allowing LRU eviction."""
+        h = sha256_hex.lower()
+        if not HEX64_RE.match(h):
+            return False
+        cur = self.db.execute('UPDATE blobs SET pinned = 0 WHERE sha256 = ?', (h,))
+        self.db.commit()
+        return cur.rowcount > 0
+
+    def is_pinned(self, sha256_hex: str) -> bool:
+        """Check if blob is pinned against LRU eviction."""
+        h = sha256_hex.lower()
+        row = self.db._one('SELECT pinned FROM blobs WHERE sha256 = ?', (h,))
+        return bool(row and row[0])
+
+    def evict_lru(self, needed_bytes: int = 0) -> int:
+        """Evict oldest unpinned blobs by accessed_at timestamp to free needed_bytes.
+
+        Returns total bytes freed.
+        """
+        rows = self.db._all(
+            'SELECT sha256, size FROM blobs WHERE pinned = 0 '
+            'ORDER BY COALESCE(accessed_at, created_at) ASC, created_at ASC'
+        )
+        freed = 0
+        for h, sz in rows:
+            if needed_bytes > 0 and freed >= needed_bytes:
+                break
+            if self.delete(h):
+                freed += sz
+        return freed
+
+    def close(self) -> None:
+        """Safely close underlying metadata database."""
+        self.db.close()