resontech 0.1.1__py3-none-any.whl

resontech/__init__.py

@@ -0,0 +1,98 @@
+"""
+ResonTech SDK — federated learning job submission for the ResonTech platform.
+
+The SDK's job is to:
+
+  1. Authenticate with the REST API.
+  2. Render every NVFlare artefact (configs + scripts + model_def.py) in memory.
+  3. Upload shards + scripts + configs directly to the user's Garage S3 bucket.
+  4. Submit the task (``POST /api/tasks``).
+  5. Return a ``Job`` handle that points at the web dashboard.
+
+Post-submit tracking (logs, progress, output files, model download) lives on
+the web UI. Open ``job.dashboard_url`` once submission returns.
+
+Quick start::
+
+    import torch.nn as nn
+    from torchvision import models
+    from resontech import ResonTech, ResonTechConfig, TrainingConfig, FederationConfig
+
+    class MyResNet(nn.Module):
+        def __init__(self, num_classes: int):
+            super().__init__()
+            self.backbone = models.resnet18(weights=None)
+            self.backbone.fc = nn.Linear(self.backbone.fc.in_features, num_classes)
+        def forward(self, x):
+            return self.backbone(x)
+
+    config = ResonTechConfig(
+        base_url="https://api.reson.tech",
+        email="you@example.com",
+        password="your-password",
+        s3_access_key_id="AKIA...",
+        s3_secret_access_key="...",
+    )
+
+    sdk = ResonTech(config)
+    sdk.login()
+    job = sdk.rt_submit(
+        model=MyResNet,
+        name="my-fl-job",
+        shards_dir="./shards",
+        training=TrainingConfig(num_classes=10, local_epochs=2),
+        federation=FederationConfig(num_rounds=5),
+    )
+    print(job.dashboard_url)
+"""
+
+from ._version import __version__
+from .client import ResonTech
+from .config import ResonTechConfig
+from .rt import RTJobBuilder, FederationConfig, ModelConfig, TrainingConfig
+from .exceptions import (
+    AuthError,
+    ResonTechError,
+    NotFoundError,
+    ServerError,
+    ValidationError,
+    StorageError,
+    WorkspaceError,
+)
+from .models.job import Job, JobState
+from .models.progress import TrainingProgress
+from .models.requests import JobSubmitRequest
+from .models.user import User
+from .models.worker import WorkerStats, WorkerSummary
+from .models.workspace import WorkspacePaths
+
+__all__ = [
+    # Core
+    "ResonTech",
+    "ResonTechConfig",
+    # Request / submission
+    "JobSubmitRequest",
+    # Models / return types
+    "Job",
+    "JobState",
+    "User",
+    "WorkspacePaths",
+    "TrainingProgress",
+    "WorkerStats",
+    "WorkerSummary",
+    # Exceptions
+    "ResonTechError",
+    "AuthError",
+    "NotFoundError",
+    "ValidationError",
+    "StorageError",
+    "WorkspaceError",
+    "ServerError",
+    # RT job building
+    "RTJobBuilder",
+    "ModelConfig",
+    "TrainingConfig",
+    "FederationConfig",
+    # Meta
+    "__version__",
+]

resontech/_http.py

@@ -0,0 +1,117 @@
+from __future__ import annotations
+
+import httpx
+
+from .exceptions import (
+    AuthError,
+    NotFoundError,
+    ResonTechError,
+    ServerError,
+    ValidationError,
+)
+
+
+class HttpClient:
+    """
+    Thin httpx wrapper for the ResonTech REST API.
+
+    - Persists cookies across requests (httpx.Client cookie jar).
+    - Sends a Bearer token in ``Authorization`` once ``set_access_token`` is called.
+    - On 401 attempts one silent refresh via ``/api/auth/refresh`` before
+      surfacing :class:`AuthError`.
+    """
+
+    def __init__(self, base_url: str, timeout: float = 60.0) -> None:
+        self._base_url = base_url.rstrip("/")
+        self._client = httpx.Client(
+            base_url=self._base_url,
+            follow_redirects=True,
+            timeout=timeout,
+        )
+        self._authed = False
+
+    # ── Auth ──────────────────────────────────────────────────────────────────
+
+    def set_access_token(self, token: str) -> None:
+        self._client.headers["Authorization"] = f"Bearer {token}"
+        self._authed = True
+
+    # ── Public request methods ────────────────────────────────────────────────
+
+    def get(self, path: str, **kwargs) -> dict | list:
+        return self._request("get", path, **kwargs)
+
+    def post(self, path: str, json: dict | None = None, **kwargs) -> dict:
+        return self._request("post", path, json=json, **kwargs)
+
+    # ── Internals ─────────────────────────────────────────────────────────────
+
+    def _request(self, method: str, path: str, **kwargs) -> dict | list:
+        try:
+            resp = getattr(self._client, method)(path, **kwargs)
+        except httpx.ConnectError as exc:
+            raise ResonTechError(
+                f"Cannot reach {self._base_url} — is the backend running?"
+            ) from exc
+
+        # One silent refresh on 401, then surface AuthError if it still fails.
+        if resp.status_code == 401 and self._authed and self._try_refresh():
+            resp = getattr(self._client, method)(path, **kwargs)
+
+        self._raise_for_status(resp)
+        # Tolerate 204/empty bodies — some endpoints (e.g. upload/complete)
+        # return no content on success.
+        if resp.status_code == 204 or not resp.content:
+            return {}
+        try:
+            return resp.json()
+        except ValueError:
+            return {}
+
+    def _try_refresh(self) -> bool:
+        """POST /api/auth/refresh using the refresh_token cookie. Returns True on success."""
+        try:
+            r = self._client.post("/api/auth/refresh")
+        except Exception:
+            return False
+        if r.status_code not in (200, 201):
+            return False
+        token = r.json().get("accessToken")
+        if not token:
+            return False
+        self.set_access_token(token)
+        return True
+
+    @staticmethod
+    def _raise_for_status(resp: httpx.Response) -> None:
+        code = resp.status_code
+        if code < 400:
+            return
+        # Prefer the JSON "message" field, fall back to the raw body.
+        try:
+            msg = resp.json().get("message", resp.text)
+        except Exception:
+            msg = resp.text or ""
+
+        if code == 400:
+            raise ValidationError(str(msg or "Validation error"))
+        if code == 401:
+            raise AuthError(f"401 Unauthorized: {msg or 'Invalid credentials or session expired.'}")
+        if code == 403:
+            raise AuthError(f"Forbidden: {msg or resp.text}")
+        if code == 404:
+            raise NotFoundError(f"Resource not found: {resp.url}")
+        if code >= 500:
+            raise ServerError(f"Server error {code}: {str(msg)[:200]}")
+        raise ResonTechError(f"HTTP {code}: {str(msg)[:200]}")
+
+    # ── Lifecycle ─────────────────────────────────────────────────────────────
+
+    def close(self) -> None:
+        self._client.close()
+
+    def __enter__(self) -> "HttpClient":
+        return self
+
+    def __exit__(self, *_: object) -> None:
+        self.close()

resontech/_s3.py

@@ -0,0 +1,252 @@
+from __future__ import annotations
+
+import os
+import sys
+import threading
+from pathlib import Path
+from typing import Optional
+
+import boto3
+from boto3.s3.transfer import TransferConfig
+from botocore.config import Config as BotoConfig
+from botocore.exceptions import BotoCoreError, ClientError
+
+from .exceptions import StorageError
+
+
+# Mirrors the web UI's upload constants (STORAGE.md):
+#   files < 10 MB → single PutObject
+#   files ≥ 10 MB → multipart, 50 MB parts, 5 concurrent parts
+_MULTIPART_THRESHOLD = 10 * 1024 * 1024
+_MULTIPART_CHUNKSIZE = 50 * 1024 * 1024
+_MAX_CONCURRENCY = 2     # was 5 — reduced for Garage stability
+
+
+class S3Client:
+    """
+    Thin boto3 wrapper pointed at the user's Garage bucket.
+
+    The underlying ``boto3`` client does the heavy lifting — multipart uploads
+    and concurrency are handled by the built-in TransferManager. This wrapper
+    adds: consistent error wrapping into :class:`StorageError`, byte/file
+    convenience methods, and a tiny terminal progress printer.
+
+    Escape hatch: the raw client is available as ``S3Client.boto3`` if you
+    need an API this wrapper does not expose (e.g. copy, delete, sync).
+    """
+
+    def __init__(
+        self,
+        *,
+        endpoint: str,
+        region: str,
+        access_key_id: str,
+        secret_access_key: str,
+        bucket: str,
+    ) -> None:
+        if not access_key_id or not secret_access_key:
+            raise StorageError(
+                "s3_access_key_id and s3_secret_access_key are required. "
+                "Provision a bucket in the web UI (Profile → Storage) to get them."
+            )
+        self._bucket = bucket
+        self._transfer = TransferConfig(
+            multipart_threshold=_MULTIPART_THRESHOLD,
+            multipart_chunksize=_MULTIPART_CHUNKSIZE,
+            max_concurrency=_MAX_CONCURRENCY,
+            use_threads=True,
+        )
+        self._client = boto3.client(
+            "s3",
+            endpoint_url=endpoint,
+            region_name=region,
+            aws_access_key_id=access_key_id,
+            aws_secret_access_key=secret_access_key,
+            config=BotoConfig(
+                signature_version="s3v4",
+                s3={"addressing_style": "path"},
+                retries={"max_attempts": 5, "mode": "standard"},
+            ),
+        )
+
+    # ── Accessors ─────────────────────────────────────────────────────────────
+
+    @property
+    def bucket(self) -> str:
+        return self._bucket
+
+    @property
+    def boto3(self):  # type: ignore[override]
+        """Escape hatch — the raw ``boto3`` S3 client. Use for ops this wrapper skips."""
+        return self._client
+
+    def set_bucket(self, bucket: str) -> None:
+        self._bucket = bucket
+
+    # ── Writes ────────────────────────────────────────────────────────────────
+
+    def put_bytes(self, key: str, body: bytes | str, content_type: str = "application/octet-stream") -> None:
+        """Write a small payload (configs, scripts) to S3."""
+        data = body.encode("utf-8") if isinstance(body, str) else body
+        try:
+            self._client.put_object(
+                Bucket=self._bucket,
+                Key=key,
+                Body=data,
+                ContentType=content_type,
+            )
+        except (BotoCoreError, ClientError) as exc:
+            raise StorageError(f"Failed to upload {key}: {exc}") from exc
+
+    def upload_file(
+        self,
+        local_path: str | os.PathLike,
+        key: str,
+        content_type: str = "application/octet-stream",
+        show_progress: bool = True,
+    ) -> None:
+        """
+        Upload a local file to S3. boto3 auto-switches to multipart for files
+        larger than :data:`_MULTIPART_THRESHOLD` (10 MB by default).
+
+        On any failure (network error or KeyboardInterrupt), best-effort aborts
+        any multipart upload that boto3 left active for this key — Garage and
+        many other S3 implementations don't auto-GC orphans, and a stale
+        multipart will block the next CreateMultipartUpload on the same key.
+        """
+        path = Path(local_path)
+        if not path.is_file():
+            raise StorageError(f"File not found: {path}")
+        size = path.stat().st_size
+        cb = _ProgressPrinter(path.name, size) if show_progress else None
+        try:
+            self._client.upload_file(
+                Filename=str(path),
+                Bucket=self._bucket,
+                Key=key,
+                Config=self._transfer,
+                ExtraArgs={"ContentType": content_type},
+                Callback=cb,
+            )
+        except (BotoCoreError, ClientError) as exc:
+            self._abort_multiparts_for_key(key)
+            raise StorageError(f"Failed to upload {path.name} → {key}: {exc}") from exc
+        except (KeyboardInterrupt, SystemExit):
+            self._abort_multiparts_for_key(key)
+            raise
+        if cb is not None:
+            cb.finish()
+
+    # ── Multipart cleanup ─────────────────────────────────────────────────────
+
+    def list_multipart_uploads(self, prefix: str = "") -> list[dict]:
+        """List active multipart uploads under ``prefix``."""
+        try:
+            resp = self._client.list_multipart_uploads(Bucket=self._bucket, Prefix=prefix)
+        except (BotoCoreError, ClientError) as exc:
+            raise StorageError(f"list_multipart_uploads({prefix!r}) failed: {exc}") from exc
+        return resp.get("Uploads", []) or []
+
+    def abort_multipart_uploads(self, prefix: str = "") -> int:
+        """
+        Abort every active multipart upload under ``prefix``. Returns the
+        number aborted. Stale multiparts left over from prior failed runs
+        block CreateMultipartUpload on the same key on Garage; sweeping at
+        submission start is cheap and removes a class of "stuck upload" bugs.
+        """
+        uploads = self.list_multipart_uploads(prefix)
+        n = 0
+        for u in uploads:
+            try:
+                self._client.abort_multipart_upload(
+                    Bucket=self._bucket, Key=u["Key"], UploadId=u["UploadId"]
+                )
+                n += 1
+            except (BotoCoreError, ClientError):
+                pass  # best-effort
+        return n
+
+    def _abort_multiparts_for_key(self, key: str) -> None:
+        """Best-effort abort of any in-flight multiparts for a single key."""
+        try:
+            for u in self.list_multipart_uploads(prefix=key):
+                if u["Key"] != key:
+                    continue
+                try:
+                    self._client.abort_multipart_upload(
+                        Bucket=self._bucket, Key=key, UploadId=u["UploadId"]
+                    )
+                except (BotoCoreError, ClientError):
+                    pass
+        except StorageError:
+            pass
+
+    # ── Reads / metadata ──────────────────────────────────────────────────────
+
+    def head(self, key: str) -> Optional[dict]:
+        """Return object metadata, or ``None`` if the key does not exist."""
+        try:
+            return self._client.head_object(Bucket=self._bucket, Key=key)
+        except ClientError as exc:
+            code = exc.response.get("Error", {}).get("Code", "")
+            if code in ("404", "NoSuchKey", "NotFound"):
+                return None
+            raise StorageError(f"head_object({key}) failed: {exc}") from exc
+
+    def list(self, prefix: str) -> list[str]:
+        """List object keys under a prefix (flat, no delimiter)."""
+        try:
+            resp = self._client.list_objects_v2(Bucket=self._bucket, Prefix=prefix)
+        except (BotoCoreError, ClientError) as exc:
+            raise StorageError(f"list({prefix}) failed: {exc}") from exc
+        return [item["Key"] for item in resp.get("Contents", [])]
+
+    def presign_get(self, key: str, expires_in: int = 3600) -> str:
+        """Generate a presigned download URL. Default TTL: 1 hour."""
+        try:
+            return self._client.generate_presigned_url(
+                "get_object",
+                Params={"Bucket": self._bucket, "Key": key},
+                ExpiresIn=expires_in,
+            )
+        except (BotoCoreError, ClientError) as exc:
+            raise StorageError(f"presign_get({key}) failed: {exc}") from exc
+
+
+class _ProgressPrinter:
+    """
+    Tiny stderr progress printer for CLI uploads. boto3 calls this from worker
+    threads, so the counter update is lock-guarded.
+
+    Notebook note: the ``\\r`` carriage return works in terminals and the
+    classic Jupyter console; some IDE renderers leave a trail (one line per
+    percent). This is cosmetic.
+    """
+
+    def __init__(self, name: str, total: int) -> None:
+        self._name = name
+        self._total = max(total, 1)
+        self._size_mb = total / (1024 * 1024)
+        self._seen = 0
+        self._lock = threading.Lock()
+        self._last_pct_step = -1
+
+    def __call__(self, bytes_amount: int) -> None:
+        with self._lock:
+            self._seen += bytes_amount
+            pct = int(self._seen * 100 / self._total)
+            # Emit a new line every 5 % so Jupyter renderers (which coalesce
+            # \r-only updates) still produce visible progress per shard.
+            step = pct // 5
+            if step != self._last_pct_step:
+                self._last_pct_step = step
+                seen_mb = self._seen / (1024 * 1024)
+                sys.stderr.write(
+                    f"[resontech] {self._name}: {pct:3d}% "
+                    f"({seen_mb:7.1f} / {self._size_mb:7.1f} MB)\n"
+                )
+                sys.stderr.flush()
+
+    def finish(self) -> None:
+        sys.stderr.write(f"[resontech] {self._name}: done\n")
+        sys.stderr.flush()

resontech/_version.py

@@ -0,0 +1 @@
+__version__ = "0.1.1"