soreplicator 0.1.0__py3-none-any.whl

soreplicator/__init__.py

@@ -0,0 +1,3 @@
+"""Simons Observatory REplicator."""
+
+__version__ = "0.1.0"

soreplicator/client/__init__.py

@@ -0,0 +1,3 @@
+"""SORE command-line client."""
+
+__version__ = "0.1.0"

soreplicator/client/cli.py

@@ -0,0 +1,154 @@
+"""The ``sore`` command-line interface."""
+
+from __future__ import annotations
+
+import argparse
+import getpass
+import sys
+from pathlib import Path
+from typing import TextIO
+from urllib.parse import urlsplit
+
+from pydantic import ValidationError
+
+from soreplicator.client.config import (
+    ClientConfig,
+    config_path,
+    load_config,
+    save_config,
+)
+from soreplicator.client.storage import StorageError, delete, upload
+
+DEFAULT_ENDPOINT = "https://sore.simonsobservatory.org"
+
+
+def _human_bytes(value: int) -> str:
+    size = float(value)
+    for unit in ("B", "KB", "MB", "GB", "TB"):
+        if size < 1000 or unit == "TB":
+            return f"{size:.0f} {unit}" if unit == "B" else f"{size:.1f} {unit}"
+        size /= 1000
+    return f"{size:.1f} TB"
+
+
+class ProgressBar:
+    def __init__(self, total: int, stream: TextIO = sys.stderr, width: int = 28):
+        self.total = total
+        self.stream = stream
+        self.width = width
+        self.enabled = stream.isatty()
+        self.rendered = False
+        if self.enabled:
+            self.update(0, total)
+
+    def update(self, uploaded: int, total: int) -> None:
+        if not self.enabled:
+            return
+        fraction = 1.0 if total == 0 else min(1.0, uploaded / total)
+        filled = round(self.width * fraction)
+        bar = "#" * filled + "-" * (self.width - filled)
+        self.stream.write(
+            f"\rUploading [{bar}] {fraction:6.1%} "
+            f"{_human_bytes(uploaded)} / {_human_bytes(total)}"
+        )
+        self.stream.flush()
+        self.rendered = True
+
+    def close(self) -> None:
+        if self.rendered:
+            self.stream.write("\n")
+            self.stream.flush()
+            self.rendered = False
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="sore", description="Upload immutable files to the SORE replica service."
+    )
+    actions = parser.add_mutually_exclusive_group(required=True)
+    actions.add_argument(
+        "--setup", action="store_true", help="store gateway credentials"
+    )
+    actions.add_argument("-i", "--input", type=Path, help="local file to upload")
+    actions.add_argument(
+        "-d", "--delete", metavar="REMOTE_PATH", help="delete an object"
+    )
+    parser.add_argument("-o", "--output", help="remote path for an upload")
+    parser.add_argument(
+        "-f", "--force", action="store_true", help="skip delete confirmation"
+    )
+    parser.add_argument(
+        "--config", type=Path, default=config_path(), help=argparse.SUPPRESS
+    )
+    return parser
+
+
+def _setup(path: Path) -> int:
+    endpoint = (
+        input(f"Gateway endpoint [{DEFAULT_ENDPOINT}]: ").strip() or DEFAULT_ENDPOINT
+    )
+    parsed = urlsplit(endpoint)
+    if parsed.scheme not in {"http", "https"} or not parsed.netloc:
+        raise ValueError("endpoint must be an absolute HTTP(S) URL")
+    hostname = parsed.hostname or ""
+    is_loopback = hostname in {"localhost", "127.0.0.1", "::1"} or hostname.endswith(
+        ".localhost"
+    )
+    if parsed.scheme != "https" and not is_loopback:
+        raise ValueError(
+            "credentials may only be sent over HTTPS (except on localhost)"
+        )
+    username = input("Username: ").strip()
+    password = getpass.getpass("Password: ")
+    if not password:
+        raise ValueError("password cannot be empty")
+    destination = save_config(
+        ClientConfig(endpoint=endpoint, username=username, password=password), path
+    )
+    print(f"Credentials saved to {destination} with mode 600.")
+    return 0
+
+
+def _confirm_delete(remote_path: str) -> bool:
+    answer = input(f"Delete {remote_path!r} permanently? [y/N] ").strip().lower()
+    return answer in {"y", "yes"}
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = build_parser()
+    args = parser.parse_args(argv)
+    try:
+        if args.setup:
+            if args.output or args.force:
+                parser.error("--setup cannot be combined with -o or -f")
+            return _setup(args.config)
+
+        config = load_config(args.config)
+        if args.input:
+            if not args.output:
+                parser.error("-o/--output is required with -i/--input")
+            if args.force:
+                parser.error("-f/--force is only valid with -d/--delete")
+            progress = ProgressBar(args.input.stat().st_size)
+            try:
+                upload(config, args.input, args.output, progress.update)
+            finally:
+                progress.close()
+            print(f"Uploaded {args.input} to sore/{config.username}/{args.output}")
+            return 0
+
+        if args.output:
+            parser.error("-o/--output is only valid with -i/--input")
+        if not args.force and not _confirm_delete(args.delete):
+            print("Delete cancelled.")
+            return 0
+        delete(config, args.delete)
+        print(f"Deleted sore/{config.username}/{args.delete}")
+        return 0
+    except (OSError, PermissionError, ValueError, ValidationError, StorageError) as exc:
+        print(f"sore: {exc}", file=sys.stderr)
+        return 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

soreplicator/client/config.py

@@ -0,0 +1,89 @@
+"""Secure local configuration storage for the SORE client."""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+import stat
+import tempfile
+from pathlib import Path
+
+from pydantic import AnyHttpUrl, BaseModel, ConfigDict, SecretStr, field_validator
+
+
+class ClientConfig(BaseModel):
+    model_config = ConfigDict(extra="forbid")
+
+    endpoint: AnyHttpUrl
+    username: str
+    password: SecretStr
+
+    @field_validator("username")
+    @classmethod
+    def normalize_username(cls, value: str) -> str:
+        value = value.strip().lower()
+        if not re.fullmatch(r"[a-z\d](?:[a-z\d]|-(?=[a-z\d])){0,38}", value):
+            raise ValueError("username is not a valid GitHub/S3 bucket name")
+        return value
+
+    @field_validator("endpoint")
+    @classmethod
+    def require_secure_endpoint(cls, value: AnyHttpUrl) -> AnyHttpUrl:
+        hostname = value.host or ""
+        is_loopback = hostname in {
+            "localhost",
+            "127.0.0.1",
+            "::1",
+        } or hostname.endswith(".localhost")
+        if value.scheme != "https" and not is_loopback:
+            raise ValueError("endpoint must use HTTPS except on localhost")
+        return value
+
+
+def config_path() -> Path:
+    base = os.environ.get("XDG_CONFIG_HOME")
+    return (Path(base) if base else Path.home() / ".config") / "sore.json"
+
+
+def save_config(config: ClientConfig, path: Path | None = None) -> Path:
+    destination = path or config_path()
+    destination.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
+    if destination.is_symlink():
+        raise ValueError(f"refusing to overwrite symlink: {destination}")
+    payload = json.dumps(
+        {
+            "endpoint": str(config.endpoint).rstrip("/"),
+            "username": config.username,
+            "password": config.password.get_secret_value(),
+        },
+        indent=2,
+    )
+    descriptor, temporary_name = tempfile.mkstemp(
+        prefix=".sore-", suffix=".tmp", dir=destination.parent, text=True
+    )
+    temporary = Path(temporary_name)
+    try:
+        os.fchmod(descriptor, 0o600)
+        with os.fdopen(descriptor, "w", encoding="utf-8") as handle:
+            handle.write(payload)
+            handle.write("\n")
+            handle.flush()
+            os.fsync(handle.fileno())
+        os.replace(temporary, destination)
+        destination.chmod(0o600)
+    finally:
+        temporary.unlink(missing_ok=True)
+    return destination
+
+
+def load_config(path: Path | None = None) -> ClientConfig:
+    source = path or config_path()
+    if source.is_symlink():
+        raise ValueError(f"refusing to read symlink: {source}")
+    mode = stat.S_IMODE(source.stat().st_mode)
+    if mode & 0o077:
+        raise PermissionError(
+            f"{source} has unsafe permissions {mode:o}; run chmod 600 {source}"
+        )
+    return ClientConfig.model_validate_json(source.read_text(encoding="utf-8"))

soreplicator/client/storage.py

@@ -0,0 +1,225 @@
+"""S3 operations used by the command-line client."""
+
+from __future__ import annotations
+
+import os
+from collections.abc import Callable
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from math import ceil
+from pathlib import Path, PurePosixPath
+
+import boto3
+from botocore.config import Config
+from botocore.exceptions import BotoCoreError, ClientError
+
+from soreplicator.client.config import ClientConfig
+
+
+class StorageError(RuntimeError):
+    pass
+
+
+ProgressCallback = Callable[[int, int], None]
+
+MIB = 1024 * 1024
+MULTIPART_THRESHOLD = 16 * MIB
+DEFAULT_PART_SIZE = 8 * MIB
+MIN_PART_SIZE = 5 * MIB
+MAX_PARTS = 10_000
+MAX_CONCURRENCY = 4
+
+
+def normalize_key(value: str) -> str:
+    if (
+        not value
+        or value.startswith("/")
+        or "\\" in value
+        or any(ord(character) < 32 or ord(character) == 127 for character in value)
+    ):
+        raise ValueError("remote path must be a non-empty relative POSIX path")
+    if any(part in {"", ".", ".."} for part in value.split("/")):
+        raise ValueError("remote path cannot contain empty, '.' or '..' components")
+    path = PurePosixPath(value)
+    normalized = str(path)
+    if len(normalized.encode("utf-8")) > 1024:
+        raise ValueError("remote path is longer than S3's 1024-byte limit")
+    return normalized
+
+
+def _client(config: ClientConfig):
+    return boto3.client(
+        "s3",
+        endpoint_url=str(config.endpoint).rstrip("/"),
+        aws_access_key_id=config.username,
+        aws_secret_access_key=config.password.get_secret_value(),
+        config=Config(
+            signature_version="s3v4",
+            s3={"addressing_style": "path"},
+            retries={"mode": "standard", "max_attempts": 4},
+        ),
+    )
+
+
+def _is_conflict(exc: ClientError) -> bool:
+    code = exc.response.get("Error", {}).get("Code", "")
+    status = exc.response.get("ResponseMetadata", {}).get("HTTPStatusCode")
+    return code in {"PreconditionFailed", "ConditionalRequestConflict"} or status in {
+        409,
+        412,
+    }
+
+
+def _is_missing(exc: ClientError) -> bool:
+    code = exc.response.get("Error", {}).get("Code", "")
+    status = exc.response.get("ResponseMetadata", {}).get("HTTPStatusCode")
+    return code in {"404", "NoSuchKey", "NotFound"} or status == 404
+
+
+def _part_size(file_size: int) -> int:
+    required = ceil(file_size / MAX_PARTS)
+    target = max(DEFAULT_PART_SIZE, MIN_PART_SIZE, required)
+    return ceil(target / MIB) * MIB
+
+
+def _upload_part(
+    client,
+    source: Path,
+    bucket: str,
+    key: str,
+    upload_id: str,
+    part_number: int,
+    offset: int,
+    length: int,
+) -> tuple[int, str, int]:
+    with source.open("rb") as handle:
+        handle.seek(offset)
+        body = handle.read(length)
+    if len(body) != length:
+        raise StorageError(f"input changed while it was being uploaded: {source}")
+    response = client.upload_part(
+        Bucket=bucket,
+        Key=key,
+        UploadId=upload_id,
+        PartNumber=part_number,
+        Body=body,
+        ContentLength=length,
+    )
+    return part_number, response["ETag"], length
+
+
+def _multipart_upload(
+    client,
+    source: Path,
+    bucket: str,
+    key: str,
+    file_size: int,
+    progress: ProgressCallback | None,
+) -> None:
+    upload_id: str | None = None
+    completed = False
+    try:
+        upload_id = client.create_multipart_upload(Bucket=bucket, Key=key)["UploadId"]
+        part_size = _part_size(file_size)
+        part_count = ceil(file_size / part_size)
+        uploaded = 0
+        parts: list[dict[str, int | str]] = []
+        workers = min(MAX_CONCURRENCY, part_count)
+        with ThreadPoolExecutor(max_workers=workers) as executor:
+            futures = []
+            for index in range(part_count):
+                offset = index * part_size
+                length = min(part_size, file_size - offset)
+                futures.append(
+                    executor.submit(
+                        _upload_part,
+                        client,
+                        source,
+                        bucket,
+                        key,
+                        upload_id,
+                        index + 1,
+                        offset,
+                        length,
+                    )
+                )
+            try:
+                for future in as_completed(futures):
+                    part_number, etag, length = future.result()
+                    parts.append({"PartNumber": part_number, "ETag": etag})
+                    uploaded += length
+                    if progress:
+                        progress(uploaded, file_size)
+            except Exception:
+                for future in futures:
+                    future.cancel()
+                raise
+
+        parts.sort(key=lambda part: int(part["PartNumber"]))
+        client.complete_multipart_upload(
+            Bucket=bucket,
+            Key=key,
+            UploadId=upload_id,
+            MultipartUpload={"Parts": parts},
+            IfNoneMatch="*",
+        )
+        completed = True
+    finally:
+        if upload_id and not completed:
+            try:
+                client.abort_multipart_upload(
+                    Bucket=bucket, Key=key, UploadId=upload_id
+                )
+            except (BotoCoreError, ClientError):
+                pass
+
+
+def upload(
+    config: ClientConfig,
+    source: Path,
+    destination: str,
+    progress: ProgressCallback | None = None,
+) -> None:
+    if not source.is_file():
+        raise ValueError(f"input is not a regular file: {source}")
+    key = normalize_key(destination)
+    file_size = source.stat().st_size
+    client = _client(config)
+    try:
+        if file_size >= MULTIPART_THRESHOLD:
+            _multipart_upload(client, source, config.username, key, file_size, progress)
+        else:
+            with source.open("rb") as handle:
+                client.put_object(
+                    Bucket=config.username,
+                    Key=key,
+                    Body=handle,
+                    ContentLength=os.fstat(handle.fileno()).st_size,
+                    IfNoneMatch="*",
+                )
+            if progress:
+                progress(file_size, file_size)
+    except ClientError as exc:
+        code = exc.response.get("Error", {}).get("Code", "")
+        detail = exc.response.get("Error", {}).get("Message", "")
+        if _is_conflict(exc):
+            raise StorageError(
+                f"{key} already exists; SORE never overwrites objects"
+            ) from exc
+        raise StorageError(f"upload failed ({code or 'gateway error'}, {detail or 'no detail'})") from exc
+    except BotoCoreError as exc:
+        raise StorageError("upload failed because the gateway is unavailable") from exc
+
+
+def delete(config: ClientConfig, destination: str) -> None:
+    key = normalize_key(destination)
+    client = _client(config)
+    try:
+        client.head_object(Bucket=config.username, Key=key)
+        client.delete_object(Bucket=config.username, Key=key)
+    except ClientError as exc:
+        if _is_missing(exc):
+            raise StorageError(f"{key} does not exist") from exc
+        code = exc.response.get("Error", {}).get("Code", "gateway error")
+        raise StorageError(f"delete failed ({code})") from exc
+    except BotoCoreError as exc:
+        raise StorageError("delete failed because the gateway is unavailable") from exc

soreplicator/server/__init__.py

@@ -0,0 +1,5 @@
+"""SORE credential provisioning web service."""
+
+from soreplicator.server.app import create_app
+
+__all__ = ["create_app"]