sqlmirror 0.2.0__py3-none-any.whl

sqlmirror/source.py

@@ -0,0 +1,117 @@
+"""Where change files come from: an S3-compatible bucket or a local folder.
+
+Layout written by the Forge app under the configured prefix:
+
+    issues/YYYY/MM/DD/HH/<epoch ms>-<random>.ndjson.gz   issue snapshots and tombstones
+    metadata/YYYY/MM/DD/<epoch ms>-<random>.json.gz      configuration snapshot
+    _sqlmirror/status.json                               app status (license, version, last sync)
+
+Keys sort in time order, so listing with StartAfter returns new files.
+"""
+from __future__ import annotations
+
+import gzip
+import json
+import os
+from pathlib import Path
+from typing import Any, Iterator
+from urllib.parse import urlparse
+
+
+class Source:
+    prefix: str = ""
+
+    def list(self, sub: str, start_after: str | None = None) -> Iterator[str]:
+        raise NotImplementedError
+
+    def read(self, key: str) -> bytes | None:
+        raise NotImplementedError
+
+    def describe(self) -> str:
+        raise NotImplementedError
+
+    def key(self, sub: str) -> str:
+        return f"{self.prefix}{sub}" if self.prefix else sub
+
+    def status(self) -> dict[str, Any] | None:
+        raw = self.read(self.key("_sqlmirror/status.json"))
+        if raw is None:
+            return None
+        try:
+            return json.loads(raw)
+        except ValueError:
+            return None
+
+
+class LocalSource(Source):
+    """A folder with the same layout as the bucket (for tests and air-gapped transfer)."""
+
+    def __init__(self, root: str | os.PathLike[str]):
+        self.root = Path(root)
+        self.prefix = ""
+
+    def list(self, sub: str, start_after: str | None = None) -> Iterator[str]:
+        base = self.root / sub
+        if not base.exists():
+            return iter(())
+        keys = sorted(str(p.relative_to(self.root)).replace(os.sep, "/") for p in base.rglob("*") if p.is_file())
+        return iter([k for k in keys if start_after is None or k > start_after])
+
+    def read(self, key: str) -> bytes | None:
+        p = self.root / key
+        return p.read_bytes() if p.exists() else None
+
+    def describe(self) -> str:
+        return str(self.root)
+
+
+class S3Source(Source):
+    """Any S3-compatible bucket: AWS S3, Cloudflare R2, Google Cloud Storage (HMAC), MinIO."""
+
+    def __init__(self, url: str, *, endpoint_url: str | None = None, region: str | None = None,
+                 profile: str | None = None):
+        import boto3  # imported lazily so local tests don't need AWS libraries configured
+
+        parsed = urlparse(url)
+        if parsed.scheme != "s3" or not parsed.netloc:
+            raise ValueError("bucket URL must look like s3://bucket/prefix")
+        self.bucket = parsed.netloc
+        prefix = parsed.path.lstrip("/")
+        self.prefix = prefix if not prefix or prefix.endswith("/") else prefix + "/"
+        session = boto3.session.Session(profile_name=profile) if profile else boto3.session.Session()
+        self.client = session.client("s3", endpoint_url=endpoint_url, region_name=region)
+
+    def list(self, sub: str, start_after: str | None = None) -> Iterator[str]:
+        kwargs: dict[str, Any] = {"Bucket": self.bucket, "Prefix": self.key(sub)}
+        if start_after:
+            kwargs["StartAfter"] = start_after
+        paginator = self.client.get_paginator("list_objects_v2")
+        for page in paginator.paginate(**kwargs):
+            for obj in page.get("Contents", []) or []:
+                yield obj["Key"]
+
+    def read(self, key: str) -> bytes | None:
+        try:
+            return self.client.get_object(Bucket=self.bucket, Key=key)["Body"].read()
+        except self.client.exceptions.NoSuchKey:
+            return None
+
+    def describe(self) -> str:
+        return f"s3://{self.bucket}/{self.prefix}"
+
+
+def open_source(location: str, **kwargs: Any) -> Source:
+    if location.startswith("s3://"):
+        return S3Source(location, **kwargs)
+    return LocalSource(location)
+
+
+def parse_records(key: str, raw: bytes) -> list[dict[str, Any]]:
+    """Decode a change file into records (one JSON object per line, optionally gzipped)."""
+    if key.endswith(".gz"):
+        raw = gzip.decompress(raw)
+    text = raw.decode("utf-8")
+    if key.endswith(".json") or key.endswith(".json.gz"):
+        obj = json.loads(text)
+        return obj if isinstance(obj, list) else [obj]
+    return [json.loads(line) for line in text.splitlines() if line.strip()]