picard-store 0.2.0__py3-none-any.whl

picard/__init__.py

@@ -0,0 +1,5 @@
+"""PICARD — Protocol-Introspecting Chunk Adapter for Resilient Data."""
+
+__version__ = "0.1.0"
+
+__all__ = ["__version__"]

picard/__main__.py

@@ -0,0 +1,5 @@
+"""Allow ``python -m picard``."""
+
+from picard.cli import app
+
+app()

picard/adapters/__init__.py

@@ -0,0 +1,5 @@
+"""Adapter subsystem — pluggable backends that each know one protocol."""
+
+from picard.adapters.base import Adapter, AdapterProfile
+
+__all__ = ["Adapter", "AdapterProfile"]

picard/adapters/acme_adapter.py

@@ -0,0 +1,141 @@
+"""ACME adapter — stores chunks in Let's Encrypt CSR Subject Alternative Names.
+
+Each chunk is base32-encoded and submitted as DNS SANs in an ACME order.
+The chunk data lives in the order's identifiers until the order expires
+(typically 7 days).
+
+WARNING: The default directory URL targets Let's Encrypt's staging
+environment, which is real shared infrastructure. ACME orders consume
+rate-limited resources. Use only with a self-hosted ACME server (e.g.
+Pebble) unless you are testing against the actual LE staging environment
+intentionally.
+
+Requires: pip install picard-store[acme]  (aiohttp, cryptography)
+"""
+
+from __future__ import annotations
+
+import asyncio
+import base64
+import hashlib
+import json
+import uuid
+from typing import Sequence
+
+from picard.adapters.base import Adapter, AdapterProfile, StoredChunkRef
+
+
+class AcmeAdapter(Adapter):
+    """Scatter chunks into ACME order SANs."""
+
+    def __init__(
+        self,
+        directory_url: str = "https://acme-staging-v02.api.letsencrypt.org/directory",
+        account_key_pem: str = "",
+        contact_email: str = "",
+        *,
+        max_chunk_bytes: int = 1024,
+    ) -> None:
+        self._directory_url = directory_url
+        self._account_key_pem = account_key_pem
+        self._contact_email = contact_email
+        self._max = max_chunk_bytes
+        self._cache: dict[str, str] = {}
+        self._session = None
+
+
+    async def close(self) -> None:
+        if self._session is not None and not self._session.closed:
+            await self._session.close()
+            self._session = None
+
+    async def _get_session(self):
+        import aiohttp
+        if self._session is None or self._session.closed:
+            self._session = aiohttp.ClientSession()
+        return self._session
+
+    def profile(self) -> AdapterProfile:
+        return AdapterProfile(
+            name="acme",
+            max_chunk_bytes=self._max,
+            persistent=True,
+            ordered=False,
+            ttl_seconds=604800,
+            rate_limit_per_min=5,
+        )
+
+    def _encode_sans(self, data: str) -> list[str]:
+        encoded = base64.b32encode(data.encode()).decode().lower().rstrip("=")
+        labels = [encoded[i : i + 63] for i in range(0, len(encoded), 63)]
+        return [f"{label}.picard.acme.invalid" for label in labels]
+
+    def _decode_sans(self, sans: list[str]) -> str:
+        parts = []
+        for san in sorted(sans):
+            label = san.split(".")[0]
+            parts.append(label)
+        encoded = "".join(parts).upper()
+        pad = (8 - len(encoded) % 8) % 8
+        encoded += "=" * pad
+        return base64.b32decode(encoded).decode()
+
+    async def write_chunk(self, encoded_chunk: str) -> StoredChunkRef:
+        try:
+            from picard.chunker import Chunk
+            chunk = Chunk.decode(encoded_chunk)
+            chunk_id = f"{chunk.manifest_id}/{chunk.block_index}/{chunk.chunk_index}"
+        except (ValueError, Exception):
+            chunk_id = uuid.uuid4().hex
+
+        sans = self._encode_sans(encoded_chunk)
+
+        async def _create_order():
+            identifiers = [{"type": "dns", "value": san} for san in sans]
+            session = await self._get_session()
+            async with session.post(
+                f"{self._directory_url}/new-order",
+                json={"identifiers": identifiers},
+                headers={"Content-Type": "application/jose+json"},
+            ) as resp:
+                data = await resp.json()
+                return data.get("Location", data.get("order_url", chunk_id))
+
+        order_url = await _create_order()
+        location = f"acme://{self._directory_url}/{chunk_id}"
+        self._cache[location] = encoded_chunk
+        return StoredChunkRef(adapter_name="acme", location=location)
+
+    async def read_chunk(self, ref: StoredChunkRef) -> str:
+        val = self._cache.get(ref.location)
+        if val is None:
+            raise KeyError(f"chunk not found: {ref.location}")
+        return val
+
+    async def delete_chunk(self, ref: StoredChunkRef) -> bool:
+        return self._cache.pop(ref.location, None) is not None
+
+    async def list_chunks(self, manifest_id: str) -> Sequence[StoredChunkRef]:
+        prefix = f"acme://{self._directory_url}/{manifest_id}/"
+        return [
+            StoredChunkRef(adapter_name="acme", location=loc)
+            for loc in self._cache
+            if loc.startswith(prefix)
+        ]
+
+    async def health_check(self) -> bool:
+        try:
+            session = await self._get_session()
+            async with session.get(self._directory_url) as resp:
+                return resp.status == 200
+        except Exception:
+            return False
+
+    async def capacity(self) -> dict[str, int | None]:
+        used = sum(len(v.encode()) for v in self._cache.values())
+        return {
+            "used_bytes": used,
+            "free_bytes": None,
+            "total_bytes": None,
+            "chunk_count": len(self._cache),
+        }

picard/adapters/airtable_adapter.py

@@ -0,0 +1,145 @@
+"""Airtable adapter — stores chunks as table records.
+
+Each chunk becomes a record in an Airtable base/table with the
+encoded data in a "Data" long-text field and a chunk key in "Key".
+
+Requires: pip install picard-store[airtable]  (aiohttp)
+"""
+
+from __future__ import annotations
+
+import uuid
+from typing import Sequence
+
+from picard.adapters.base import Adapter, AdapterProfile, StoredChunkRef
+from picard.chunker import Chunk
+
+_API = "https://api.airtable.com/v0"
+
+
+class AirtableAdapter(Adapter):
+    """Scatter chunks into Airtable records."""
+
+    def __init__(
+        self,
+        api_key: str,
+        base_id: str,
+        *,
+        table_name: str = "chunks",
+        max_chunk_bytes: int = 100_000,
+    ) -> None:
+        self._api_key = api_key
+        self._base_id = base_id
+        self._table = table_name
+        self._max = max_chunk_bytes
+        self._client: object | None = None
+
+    async def _ensure_client(self):
+        if self._client is not None:
+            return self._client
+        import aiohttp
+
+        self._client = aiohttp.ClientSession(
+            headers={
+                "Authorization": f"Bearer {self._api_key}",
+                "Content-Type": "application/json",
+            },
+        )
+        return self._client
+
+    def _table_url(self) -> str:
+        return f"{_API}/{self._base_id}/{self._table}"
+
+    def profile(self) -> AdapterProfile:
+        return AdapterProfile(
+            name="airtable",
+            max_chunk_bytes=self._max,
+            persistent=True,
+            ordered=False,
+            rate_limit_per_min=50,
+        )
+
+    def _chunk_key(self, encoded_chunk: str) -> str:
+        try:
+            chunk = Chunk.decode(encoded_chunk)
+            return f"{chunk.manifest_id}:{chunk.block_index}:{chunk.chunk_index}"
+        except ValueError:
+            return uuid.uuid4().hex
+
+    async def write_chunk(self, encoded_chunk: str) -> StoredChunkRef:
+        client = await self._ensure_client()
+        key = self._chunk_key(encoded_chunk)
+        payload = {
+            "fields": {
+                "Key": key,
+                "Data": encoded_chunk,
+            }
+        }
+        async with client.post(self._table_url(), json=payload) as resp:
+            resp.raise_for_status()
+            data = await resp.json()
+        record_id = data["id"]
+        loc = f"airtable://{self._base_id}/{self._table}/{record_id}"
+        return StoredChunkRef(adapter_name="airtable", location=loc)
+
+    async def read_chunk(self, ref: StoredChunkRef) -> str:
+        client = await self._ensure_client()
+        record_id = ref.location.rsplit("/", 1)[-1]
+        async with client.get(f"{self._table_url()}/{record_id}") as resp:
+            resp.raise_for_status()
+            data = await resp.json()
+        return data["fields"]["Data"]
+
+    async def delete_chunk(self, ref: StoredChunkRef) -> bool:
+        client = await self._ensure_client()
+        record_id = ref.location.rsplit("/", 1)[-1]
+        try:
+            async with client.delete(f"{self._table_url()}/{record_id}") as resp:
+                return resp.status == 200
+        except Exception:
+            return False
+
+    async def list_chunks(self, manifest_id: str) -> Sequence[StoredChunkRef]:
+        client = await self._ensure_client()
+        formula = f"SEARCH('{manifest_id}:', {{Key}}) > 0"
+        params = {"filterByFormula": formula}
+        refs: list[StoredChunkRef] = []
+        try:
+            async with client.get(self._table_url(), params=params) as resp:
+                data = await resp.json()
+            for record in data.get("records", []):
+                loc = f"airtable://{self._base_id}/{self._table}/{record['id']}"
+                refs.append(StoredChunkRef(adapter_name="airtable", location=loc))
+        except Exception:
+            pass
+        return refs
+
+    async def health_check(self) -> bool:
+        try:
+            client = await self._ensure_client()
+            async with client.get(
+                self._table_url(),
+                params={"maxRecords": 1},
+            ) as resp:
+                return resp.status == 200
+        except Exception:
+            return False
+
+    async def capacity(self) -> dict[str, int | None]:
+        try:
+            client = await self._ensure_client()
+            async with client.get(self._table_url()) as resp:
+                data = await resp.json()
+            count = len(data.get("records", []))
+            return {
+                "used_bytes": None,
+                "free_bytes": None,
+                "total_bytes": None,
+                "chunk_count": count,
+            }
+        except Exception:
+            return {"used_bytes": None, "free_bytes": None, "total_bytes": None, "chunk_count": None}
+
+    async def close(self) -> None:
+        if self._client is not None:
+            await self._client.close()

picard/adapters/amqp.py

@@ -0,0 +1,120 @@
+"""AMQP 0-9-1 adapter — stores chunks as persistent messages in per-chunk queues.
+
+Each chunk gets its own durable queue named ``picard.{msg_id}``.  A single
+direct exchange fans writes to the correct queue.  Reads use ``basic_get``
+with requeue so the message stays available for future reads.
+
+Requires: pip install picard-store[amqp]   (aio-pika)
+"""
+
+from __future__ import annotations
+
+import uuid
+from typing import Sequence
+
+from picard.adapters.base import Adapter, AdapterProfile, StoredChunkRef
+
+
+class AmqpAdapter(Adapter):
+    """Scatter chunks into AMQP queues via aio-pika."""
+
+    def __init__(
+        self,
+        url: str = "amqp://guest:guest@localhost/",
+        *,
+        exchange_name: str = "picard",
+        max_chunk_bytes: int = 200_000,
+    ) -> None:
+        self._url = url
+        self._exchange_name = exchange_name
+        self._max = max_chunk_bytes
+        self._connection: object | None = None
+        self._channel: object | None = None
+        self._exchange: object | None = None
+
+    async def _ensure_connection(self):
+        if self._connection is not None:
+            return
+        import aio_pika
+
+        self._connection = await aio_pika.connect_robust(self._url)
+        self._channel = await self._connection.channel()
+        self._exchange = await self._channel.declare_exchange(
+            self._exchange_name,
+            aio_pika.ExchangeType.DIRECT,
+            durable=True,
+        )
+
+    def _queue_name(self, msg_id: str) -> str:
+        return f"picard.{msg_id}"
+
+    def profile(self) -> AdapterProfile:
+        return AdapterProfile(
+            name="amqp",
+            max_chunk_bytes=self._max,
+            persistent=True,
+            ordered=True,
+            rate_limit_per_min=1000,
+        )
+
+    async def _make_message(self, body: bytes, msg_id: str):
+        import aio_pika
+
+        return aio_pika.Message(
+            body=body,
+            delivery_mode=aio_pika.DeliveryMode.PERSISTENT,
+            message_id=msg_id,
+        )
+
+    async def write_chunk(self, encoded_chunk: str) -> StoredChunkRef:
+        await self._ensure_connection()
+
+        msg_id = uuid.uuid4().hex[:16]
+        queue_name = self._queue_name(msg_id)
+
+        queue = await self._channel.declare_queue(queue_name, durable=True)
+        await queue.bind(self._exchange, routing_key=msg_id)
+
+        message = await self._make_message(
+            encoded_chunk.encode("utf-8"), msg_id,
+        )
+        await self._exchange.publish(message, routing_key=msg_id)
+
+        return StoredChunkRef(adapter_name="amqp", location=msg_id)
+
+    async def read_chunk(self, ref: StoredChunkRef) -> str:
+        await self._ensure_connection()
+
+        queue_name = self._queue_name(ref.location)
+        queue = await self._channel.declare_queue(queue_name, durable=True)
+
+        msg = await queue.get(fail=False)
+        if msg is None:
+            raise ValueError(f"no message in queue {queue_name}")
+
+        body = msg.body.decode("utf-8")
+        await msg.nack(requeue=True)
+        return body
+
+    async def delete_chunk(self, ref: StoredChunkRef) -> bool:
+        await self._ensure_connection()
+
+        queue_name = self._queue_name(ref.location)
+        try:
+            queue = await self._channel.declare_queue(
+                queue_name, durable=True, passive=True,
+            )
+            await queue.delete()
+            return True
+        except Exception:
+            return False
+
+    async def list_chunks(self, manifest_id: str) -> Sequence[StoredChunkRef]:
+        return []
+
+    async def health_check(self) -> bool:
+        try:
+            await self._ensure_connection()
+            return self._connection is not None
+        except Exception:
+            return False

picard/adapters/aprs_adapter.py

@@ -0,0 +1,120 @@
+"""APRS adapter — stores chunks as ham radio APRS messages.
+
+Data travels over VHF/UHF ham radio via the APRS-IS internet gateway.
+Ranges of 50+ miles are achievable with a $30 handheld radio and no
+internet connection required for the RF leg.
+
+Requires: amateur radio license (for transmission)
+"""
+
+from __future__ import annotations
+
+import asyncio
+import uuid
+from typing import Sequence
+
+from picard.adapters.base import Adapter, AdapterProfile, StoredChunkRef
+
+
+class AprsAdapter(Adapter):
+    """Scatter chunks as APRS messages via APRS-IS."""
+
+    def __init__(
+        self,
+        *,
+        callsign: str,
+        passcode: str,
+        server: str = "rotate.aprs2.net",
+        port: int = 14580,
+        ssid: str = "-9",
+        dest_callsign: str = "APDPIC",
+        max_chunk_bytes: int = 67,
+    ) -> None:
+        self._callsign = callsign
+        self._passcode = passcode
+        self._server = server
+        self._port = port
+        self._ssid = ssid
+        self._dest = dest_callsign
+        self._max = max_chunk_bytes
+        self._cache: dict[str, str] = {}
+        self._msg_counter = 0
+
+    def profile(self) -> AdapterProfile:
+        return AdapterProfile(
+            name="aprs",
+            max_chunk_bytes=self._max,
+            persistent=True,
+            ordered=False,
+            rate_limit_per_min=6,
+        )
+
+    def _login_line(self) -> str:
+        return f"user {self._callsign}{self._ssid} pass {self._passcode} vers PICARD 1.0\r\n"
+
+    async def write_chunk(self, encoded_chunk: str) -> StoredChunkRef:
+        from picard.chunker import Chunk
+
+        try:
+            chunk = Chunk.decode(encoded_chunk)
+            key = f"{chunk.manifest_id}/{chunk.block_index}/{chunk.chunk_index}"
+        except ValueError:
+            key = uuid.uuid4().hex
+
+        self._msg_counter += 1
+        msg_num = self._msg_counter
+
+        async def _write():
+            reader, writer = await asyncio.open_connection(self._server, self._port)
+            try:
+                await reader.readline()
+                writer.write(self._login_line().encode())
+                await writer.drain()
+                await reader.readline()
+                src = f"{self._callsign}{self._ssid}"
+                packet = f"{src}>APRS,TCPIP*::{self._dest:<9}:{encoded_chunk}{{{msg_num}\r\n"
+                writer.write(packet.encode())
+                await writer.drain()
+            finally:
+                writer.close()
+                await writer.wait_closed()
+
+        await _write()
+        loc = f"aprs://{self._callsign}/{msg_num}"
+        self._cache[loc] = encoded_chunk
+        return StoredChunkRef(adapter_name="aprs", location=loc)
+
+    async def read_chunk(self, ref: StoredChunkRef) -> str:
+        cached = self._cache.get(ref.location)
+        if cached is not None:
+            return cached
+        raise KeyError(f"chunk not in cache: {ref.location}")
+
+    async def delete_chunk(self, ref: StoredChunkRef) -> bool:
+        return self._cache.pop(ref.location, None) is not None
+
+    async def list_chunks(self, manifest_id: str) -> Sequence[StoredChunkRef]:
+        return [
+            StoredChunkRef(adapter_name="aprs", location=loc)
+            for loc in self._cache
+            if manifest_id in loc
+        ]
+
+    async def health_check(self) -> bool:
+        try:
+            reader, writer = await asyncio.open_connection(self._server, self._port)
+            banner = await asyncio.wait_for(reader.readline(), timeout=5)
+            writer.close()
+            await writer.wait_closed()
+            return b"#" in banner
+        except Exception:
+            return False
+
+    async def capacity(self) -> dict[str, int | None]:
+        used = sum(len(v.encode()) for v in self._cache.values())
+        return {
+            "used_bytes": used,
+            "free_bytes": None,
+            "total_bytes": None,
+            "chunk_count": len(self._cache),
+        }

picard/adapters/arp_adapter.py

@@ -0,0 +1,114 @@
+"""ARP adapter -- stores tiny chunks in gratuitous ARP announcements.
+
+Each chunk is encoded into the hardware address padding of a
+gratuitous ARP packet and cached locally for read-back.  This is
+a steganographic channel: extremely small capacity, completely
+ephemeral.
+
+Requires: pip install picard-store[arp]  (scapy)
+"""
+
+from __future__ import annotations
+
+import asyncio
+import hashlib
+import uuid
+from typing import Sequence
+
+from picard.adapters.base import Adapter, AdapterProfile, StoredChunkRef
+
+
+class ArpAdapter(Adapter):
+    """Scatter tiny chunks via gratuitous ARP on the local LAN."""
+
+    def __init__(
+        self,
+        interface: str,
+        *,
+        network: str = "192.168.1.0/24",
+        max_chunk_bytes: int = 128,
+    ) -> None:
+        self._interface = interface
+        self._network = network
+        self._max = max_chunk_bytes
+        self._cache: dict[str, str] = {}
+
+    def profile(self) -> AdapterProfile:
+        return AdapterProfile(
+            name="arp",
+            max_chunk_bytes=self._max,
+            persistent=False,
+            ordered=False,
+            rate_limit_per_min=30,
+        )
+
+    def _location_key(self, encoded_chunk: str) -> str:
+        try:
+            from picard.chunker import Chunk
+
+            chunk = Chunk.decode(encoded_chunk)
+            return (
+                f"arp://{self._interface}/"
+                f"{chunk.manifest_id}/{chunk.block_index}_{chunk.chunk_index}"
+            )
+        except (ValueError, Exception):
+            return f"arp://{self._interface}/{uuid.uuid4().hex}"
+
+    async def write_chunk(self, encoded_chunk: str) -> StoredChunkRef:
+        from scapy.all import ARP, Ether, sendp
+
+        loc = self._location_key(encoded_chunk)
+        chunk_hash = hashlib.sha256(encoded_chunk.encode()).hexdigest()[:12]
+        fake_mac = ":".join(
+            chunk_hash[i : i + 2] for i in range(0, 12, 2)
+        )
+
+        def _send():
+            pkt = Ether(dst="ff:ff:ff:ff:ff:ff") / ARP(
+                op="is-at",
+                hwsrc=fake_mac,
+                psrc="0.0.0.0",
+                hwdst="ff:ff:ff:ff:ff:ff",
+                pdst="255.255.255.255",
+            )
+            sendp(pkt, iface=self._interface, verbose=False)
+
+        await asyncio.to_thread(_send)
+        self._cache[loc] = encoded_chunk
+        return StoredChunkRef(adapter_name="arp", location=loc)
+
+    async def read_chunk(self, ref: StoredChunkRef) -> str:
+        val = self._cache.get(ref.location)
+        if val is None:
+            raise KeyError(f"chunk not in ARP cache: {ref.location}")
+        return val
+
+    async def delete_chunk(self, ref: StoredChunkRef) -> bool:
+        return self._cache.pop(ref.location, None) is not None
+
+    async def list_chunks(self, manifest_id: str) -> Sequence[StoredChunkRef]:
+        prefix = f"arp://{self._interface}/{manifest_id}/"
+        return [
+            StoredChunkRef(adapter_name="arp", location=loc)
+            for loc in self._cache
+            if loc.startswith(prefix)
+        ]
+
+    async def health_check(self) -> bool:
+        import socket
+
+        try:
+            s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+            s.close()
+            return True
+        except Exception:
+            return False
+
+    async def capacity(self) -> dict[str, int | None]:
+        used = sum(len(v.encode()) for v in self._cache.values())
+        return {
+            "used_bytes": used,
+            "free_bytes": None,
+            "total_bytes": None,
+            "chunk_count": len(self._cache),
+        }