remote-upload 0.1.0__py3-none-any.whl

remote_upload/__init__.py

@@ -0,0 +1,93 @@
+"""remote-upload: stream local content INTO remote storage through one tiny API.
+
+Write-side twin of ``remote-download``: push bytes from your process into S3 /
+MinIO, Azure Blob, GCS, SFTP or an authenticated HTTP endpoint via a single
+framework-agnostic facade.
+
+    from remote_upload import RemoteUpload
+
+    result = (
+        RemoteUpload.to("https://api.example.com/files/report.pdf")
+        .body(data)
+        .content_type("application/pdf")
+        .upload()
+    )
+
+The ``HttpTarget`` backend is pure stdlib and always available. The cloud / SSH
+backends need an extra:
+
+    S3Target       -> pip install "remote-upload[s3]"      (boto3)
+    AzureBlobTarget-> pip install "remote-upload[azure]"   (azure-storage-blob)
+    GcsTarget      -> pip install "remote-upload[gcs]"     (google-cloud-storage)
+    SftpTarget     -> pip install "remote-upload[sftp]"    (paramiko)
+    HttpxTarget    -> pip install "remote-upload[httpx]"   (httpx; retries/auth/proxy)
+
+They are importable from the package root lazily (``from remote_upload import
+S3Target``) or directly (``from remote_upload.targets.s3 import S3Target``).
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from .content import UploadContent
+from .exceptions import RemoteUploadError, RetryableUploadError, TerminalUploadError
+from .facade import RemoteUpload
+from .progress import ProgressListener
+from .request import RemoteUploadRequest
+from .result import UploadResult
+from .target import UploadTarget
+from .targets.http import HttpTarget
+
+__version__ = "0.1.0"
+
+if TYPE_CHECKING:
+    from .targets.azure import AzureBlobTarget
+    from .targets.gcs import GcsTarget
+    from .targets.httpx_target import HttpxTarget
+    from .targets.s3 import S3Target
+    from .targets.sftp import SftpTarget
+
+# Optional, extra-gated targets: module path + the extra that ships their dep.
+_LAZY: dict[str, tuple[str, str]] = {
+    "S3Target": (".targets.s3", "s3"),
+    "AzureBlobTarget": (".targets.azure", "azure"),
+    "GcsTarget": (".targets.gcs", "gcs"),
+    "SftpTarget": (".targets.sftp", "sftp"),
+    "HttpxTarget": (".targets.httpx_target", "httpx"),
+}
+
+__all__ = [
+    "RemoteUpload",
+    "RemoteUploadRequest",
+    "UploadTarget",
+    "UploadContent",
+    "UploadResult",
+    "ProgressListener",
+    "RemoteUploadError",
+    "RetryableUploadError",
+    "TerminalUploadError",
+    "HttpTarget",
+    "S3Target",
+    "AzureBlobTarget",
+    "GcsTarget",
+    "SftpTarget",
+    "HttpxTarget",
+    "__version__",
+]
+
+
+def __getattr__(name: str) -> object:
+    """Lazily import the extra-gated targets, with a helpful error if missing."""
+    if name in _LAZY:
+        import importlib
+
+        module_path, extra = _LAZY[name]
+        try:
+            module = importlib.import_module(module_path, __name__)
+        except ImportError as exc:
+            raise ImportError(
+                f'{name} requires an optional dependency. Install it with: pip install "remote-upload[{extra}]"'
+            ) from exc
+        return getattr(module, name)
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

remote_upload/content.py

@@ -0,0 +1,36 @@
+"""The payload handed to an :class:`~remote_upload.target.UploadTarget`."""
+
+from __future__ import annotations
+
+from collections.abc import Mapping
+from dataclasses import dataclass, field
+from typing import IO
+
+
+@dataclass(frozen=True, slots=True)
+class UploadContent:
+    """The body stream plus the metadata a destination may persist.
+
+    Built by :class:`~remote_upload.request.RemoteUploadRequest` and handed to a
+    target's ``upload``. The ``body`` is owned by the caller (the request), which
+    wraps it for metering / checksum and closes it after the upload -- targets
+    read it but must **not** close it.
+    """
+
+    #: Live body to stream into the destination. Read by the target, closed by
+    #: the caller.
+    body: IO[bytes]
+
+    #: Content length in bytes, or ``None`` when unknown. Some targets (S3 single
+    #: PUT) need it; others (chunked HTTP) can stream without it.
+    content_length: int | None = None
+
+    #: MIME type to store with the object, or ``None``.
+    content_type: str | None = None
+
+    #: Suggested filename / object key tail, or ``None``.
+    filename: str | None = None
+
+    #: Arbitrary user metadata to attach to the object. Never ``None`` -- an
+    #: empty mapping when none was supplied.
+    metadata: Mapping[str, str] = field(default_factory=dict)

remote_upload/exceptions.py

@@ -0,0 +1,40 @@
+"""Exception hierarchy for remote-upload.
+
+Targets translate provider failures into one of two subtypes so callers can act
+on the distinction without parsing messages:
+
+* :class:`RetryableUploadError` -- transient (network blip, 5xx, timeout):
+  retrying later may succeed.
+* :class:`TerminalUploadError` -- permanent (auth, 4xx, quota, validation):
+  retrying the same request will fail again; fix something first.
+
+This retryable/terminal split is the deliberate improvement over a single
+exception type: it lets an outbox / sync coordinator decide between "keep
+retrying" and "mark failed, surface to user".
+"""
+
+from __future__ import annotations
+
+
+class RemoteUploadError(Exception):
+    """Base error for failures raised while streaming a body to a destination.
+
+    Prefer raising one of the two subtypes (:class:`RetryableUploadError` /
+    :class:`TerminalUploadError`) so callers can branch on retry semantics.
+    """
+
+
+class RetryableUploadError(RemoteUploadError):
+    """A transient upload failure -- a network blip, a 5xx response, a timeout.
+
+    Callers with a retry budget (an offline outbox, a sync coordinator) should
+    re-enqueue with backoff when they catch this.
+    """
+
+
+class TerminalUploadError(RemoteUploadError):
+    """A permanent upload failure -- invalid credentials, a 4xx, quota, validation.
+
+    Retrying the same request will fail again; the caller must change something
+    (re-auth, fix the payload, escalate) instead of blindly retrying.
+    """

remote_upload/facade.py

@@ -0,0 +1,37 @@
+"""Public, framework-agnostic entry point of the library."""
+
+from __future__ import annotations
+
+from .request import RemoteUploadRequest
+from .target import UploadTarget
+
+
+class RemoteUpload:
+    """Entry point: start a fluent upload request.
+
+    ::
+
+        # 1. Plain HTTP PUT to a URL
+        RemoteUpload.to("https://api.example.com/files/report.pdf") \\
+            .body(data).content_type("application/pdf").upload()
+
+        # 2. Cloud storage target (S3 / MinIO, Azure, GCS, ...)
+        target = S3Target(bucket="my-bucket", key="tenant/123/photo.jpg",
+                          endpoint="http://localhost:9000",
+                          access_key=ak, secret_key=sk)
+        result = RemoteUpload.to(target).body(data).content_type("image/jpeg").upload()
+    """
+
+    @staticmethod
+    def to(target: UploadTarget | str) -> RemoteUploadRequest:
+        """Start a request against a target, or a URL for a plain HTTP PUT.
+
+        A ``str`` is treated as an absolute HTTP/HTTPS URL and wrapped in a
+        default :class:`~remote_upload.targets.http.HttpTarget` (PUT, no auth).
+        Any other object is used directly as the :class:`UploadTarget`.
+        """
+        if isinstance(target, str):
+            from .targets.http import HttpTarget
+
+            return RemoteUploadRequest(HttpTarget(target))
+        return RemoteUploadRequest(target)

remote_upload/metered.py

@@ -0,0 +1,91 @@
+"""Stream decorator that instruments the body as the target reads it."""
+
+from __future__ import annotations
+
+import hashlib
+import io
+from typing import IO
+
+from .progress import ProgressListener
+
+
+def _new_digest(algorithm: str) -> hashlib._Hash:
+    """Build a hashlib digest, tolerating Java-style names like ``"SHA-256"``.
+
+    Tries the name verbatim first, then a normalized form (lower-cased, dashes
+    removed) so both ``"sha256"`` and ``"SHA-256"`` work.
+
+    :raises ValueError: if the algorithm is not supported by hashlib.
+    """
+    try:
+        return hashlib.new(algorithm)
+    except ValueError:
+        normalized = algorithm.lower().replace("-", "")
+        try:
+            return hashlib.new(normalized)
+        except ValueError as exc:
+            raise ValueError(f"Unsupported digest algorithm: {algorithm!r}") from exc
+
+
+class MeteredReader(io.RawIOBase):
+    """A read-only stream wrapper that instruments the body as it is consumed.
+
+    Counts bytes, optionally computes a digest, and fires a
+    :data:`~remote_upload.progress.ProgressListener` -- all driven by the target
+    SDK pulling bytes (uploads are reader-driven, so we instrument the stream the
+    SDK reads rather than running our own copy loop).
+
+    Closing this reader closes the wrapped stream (the request opens it inside a
+    ``with`` block). Targets read but must not close it.
+    """
+
+    def __init__(
+        self,
+        raw: IO[bytes],
+        total: int | None = None,
+        listener: ProgressListener | None = None,
+        digest_algorithm: str | None = None,
+    ) -> None:
+        super().__init__()
+        self._raw = raw
+        self._total = total
+        self._listener = listener
+        self._digest = _new_digest(digest_algorithm) if digest_algorithm else None
+        self._count = 0
+
+    def readable(self) -> bool:
+        return True
+
+    def readinto(self, b: bytearray | memoryview) -> int:  # type: ignore[override]
+        view = memoryview(b)
+        chunk = self._raw.read(len(view))
+        if not chunk:
+            return 0
+        n = len(chunk)
+        view[:n] = chunk
+        self._count += n
+        if self._digest is not None:
+            self._digest.update(chunk)
+        # Mirror the Java contract: fire progress on bulk reads only, never on
+        # single-byte reads (a 1-byte buffer) -- those would be far too chatty.
+        if self._listener is not None and len(view) > 1:
+            self._listener(self._count, self._total)
+        return n
+
+    @property
+    def bytes_transferred(self) -> int:
+        """Total bytes read through this stream so far."""
+        return self._count
+
+    @property
+    def checksum_hex(self) -> str | None:
+        """Lower-case hex digest, or ``None`` if no algorithm was set."""
+        return self._digest.hexdigest() if self._digest is not None else None
+
+    def close(self) -> None:
+        if self.closed:
+            return
+        try:
+            self._raw.close()
+        finally:
+            super().close()

remote_upload/progress.py

@@ -0,0 +1,18 @@
+"""Progress callback type."""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+
+#: Callback invoked as the body is streamed into the destination.
+#:
+#: Receives ``(bytes_transferred, total_bytes)`` where ``total_bytes`` is
+#: ``None`` when the caller did not supply a content length (chunked uploads,
+#: unknown-length streams)::
+#:
+#:     def on_progress(sent: int, total: int | None) -> None:
+#:         pct = (sent * 100 // total) if total else -1
+#:         print(f"uploaded {sent} / {total} bytes ({pct}%)")
+#:
+#:     RemoteUpload.to(target).body(data).on_progress(on_progress).upload()
+ProgressListener = Callable[[int, "int | None"], None]

remote_upload/request.py

@@ -0,0 +1,148 @@
+"""Fluent request that wraps a target and pushes a body into it."""
+
+from __future__ import annotations
+
+import io
+import mimetypes
+import os
+import time
+from collections.abc import Mapping
+from dataclasses import replace
+from datetime import timedelta
+from typing import IO, cast
+
+from .content import UploadContent
+from .metered import MeteredReader
+from .progress import ProgressListener
+from .result import UploadResult
+from .target import UploadTarget
+
+
+class RemoteUploadRequest:
+    """Fluent request that wraps an :class:`UploadTarget` and pushes a body into it.
+
+    Supply the body with :meth:`body` / :meth:`body_file`, decorate it with
+    content type / filename / metadata / progress / checksum, then call
+    :meth:`upload`.
+
+    :meth:`upload` consumes and **closes** the body stream. Build a new request
+    per upload -- instances are not reusable.
+    """
+
+    def __init__(self, target: UploadTarget) -> None:
+        self._target = target
+        self._body: IO[bytes] | None = None
+        self._content_length: int | None = None
+        self._content_type: str | None = None
+        self._filename: str | None = None
+        self._metadata: dict[str, str] = {}
+        self._progress_listener: ProgressListener | None = None
+        self._checksum_algorithm: str | None = None
+
+    def body(
+        self,
+        data: bytes | bytearray | memoryview | IO[bytes],
+        length: int | None = None,
+    ) -> RemoteUploadRequest:
+        """Set the body from raw bytes or an open binary stream.
+
+        * ``bytes`` / ``bytearray`` / ``memoryview`` -- the content length is
+          exact and ``length`` is ignored.
+        * a binary stream (anything with ``.read``) -- pass ``length`` when the
+          size is known (preferred: cloud targets like S3 need it); omit it for
+          chunked / unknown-length uploads.
+        """
+        if isinstance(data, (bytes, bytearray, memoryview)):
+            raw = bytes(data)
+            self._body = io.BytesIO(raw)
+            self._content_length = len(raw)
+        elif hasattr(data, "read"):
+            self._body = data
+            self._content_length = length
+        else:
+            raise TypeError(
+                f"body() expects bytes or a binary stream; got {type(data).__name__}. For a file path use body_file()."
+            )
+        return self
+
+    def body_file(self, path: str | os.PathLike[str]) -> RemoteUploadRequest:
+        """Set the body from a file on disk.
+
+        Infers the content length, and (unless already set) the filename and
+        content type.
+        """
+        self._body = open(path, "rb")  # noqa: SIM115 -- closed by upload()
+        self._content_length = os.path.getsize(path)
+        if self._filename is None:
+            self._filename = os.path.basename(os.fspath(path))
+        if self._content_type is None:
+            self._content_type = mimetypes.guess_type(os.fspath(path))[0]
+        return self
+
+    def content_type(self, content_type: str | None) -> RemoteUploadRequest:
+        """Set the MIME type stored with the object."""
+        self._content_type = content_type
+        return self
+
+    def filename(self, filename: str | None) -> RemoteUploadRequest:
+        """Set the suggested filename / object key tail."""
+        self._filename = filename
+        return self
+
+    def metadata(self, metadata: Mapping[str, str] | None) -> RemoteUploadRequest:
+        """Replace the user metadata with a copy of ``metadata`` (``None`` clears)."""
+        self._metadata = dict(metadata) if metadata else {}
+        return self
+
+    def add_metadata(self, key: str, value: str) -> RemoteUploadRequest:
+        """Add a single metadata entry."""
+        self._metadata[key] = value
+        return self
+
+    def on_progress(self, listener: ProgressListener | None) -> RemoteUploadRequest:
+        """Register a progress callback fired as the destination reads the body."""
+        self._progress_listener = listener
+        return self
+
+    def checksum(self, algorithm: str | None) -> RemoteUploadRequest:
+        """Compute a checksum of the streamed bytes, exposed in the result.
+
+        Common values: ``"md5"``, ``"sha256"`` (Java-style ``"SHA-256"`` also
+        works). Pass ``None`` to disable.
+        """
+        self._checksum_algorithm = algorithm
+        return self
+
+    def upload(self) -> UploadResult:
+        """Stream the body into the target and return the enriched result.
+
+        :raises ValueError: if no body was supplied.
+        """
+        if self._body is None:
+            raise ValueError("body is required; call body(...) before upload()")
+
+        start = time.perf_counter()
+        with MeteredReader(
+            self._body,
+            self._content_length,
+            self._progress_listener,
+            self._checksum_algorithm,
+        ) as metered:
+            content = UploadContent(
+                # MeteredReader is a readable binary stream; cast to the IO[bytes]
+                # contract (io.RawIOBase is not nominally IO[bytes] for mypy).
+                body=cast("IO[bytes]", metered),
+                content_length=self._content_length,
+                content_type=self._content_type,
+                filename=self._filename,
+                metadata=dict(self._metadata),
+            )
+            provider_result = self._target.upload(content)
+            elapsed = timedelta(seconds=time.perf_counter() - start)
+            return replace(
+                provider_result,
+                bytes_transferred=metered.bytes_transferred,
+                duration=elapsed,
+                checksum_algorithm=self._checksum_algorithm,
+                checksum_hex=metered.checksum_hex,
+            )

remote_upload/result.py

@@ -0,0 +1,57 @@
+"""Outcome of an upload."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from datetime import timedelta
+
+
+@dataclass(frozen=True, slots=True)
+class UploadResult:
+    """Outcome of a :meth:`~remote_upload.request.RemoteUploadRequest.upload` call.
+
+    Couples the numeric transfer stats (bytes, duration, optional checksum)
+    computed by the library with the provider-specific identifiers (key,
+    location, ETag, version id) returned by the target.
+
+    The target builds it with what it knows; the request enriches a copy (via
+    :func:`dataclasses.replace`) with the metered byte count, duration and
+    checksum.
+    """
+
+    #: Object key / remote path the bytes were written to.
+    key: str
+
+    #: Fully-qualified location (URL / URI) when the provider exposes one.
+    location: str | None = None
+
+    #: Provider ETag (S3 / Azure) when available.
+    etag: str | None = None
+
+    #: Provider version id when versioning is enabled.
+    version_id: str | None = None
+
+    #: Total number of bytes streamed to the destination.
+    bytes_transferred: int = 0
+
+    #: Wall-clock time spent on the upload.
+    duration: timedelta | None = None
+
+    #: Content type stored with the object.
+    content_type: str | None = None
+
+    #: Checksum algorithm computed over the body, or ``None`` if none requested.
+    checksum_algorithm: str | None = None
+
+    #: Lower-case hex checksum digest, or ``None`` if none requested.
+    checksum_hex: str | None = None
+
+    @property
+    def bytes_per_second(self) -> int:
+        """Effective throughput in bytes/second; ``0`` if duration is zero/None."""
+        if self.duration is None:
+            return 0
+        millis = self.duration / timedelta(milliseconds=1)
+        if millis <= 0:
+            return 0
+        return int(self.bytes_transferred * 1000 / millis)

remote_upload/target.py

@@ -0,0 +1,30 @@
+"""The upload port: a destination that can receive a stream of bytes."""
+
+from __future__ import annotations
+
+from typing import Protocol, runtime_checkable
+
+from .content import UploadContent
+from .result import UploadResult
+
+
+@runtime_checkable
+class UploadTarget(Protocol):
+    """Universal abstraction for any remote destination that receives bytes.
+
+    Each backend (HTTP, S3, Azure, GCS, SFTP) supplies its own
+    implementation; consumers push bytes through the same API regardless of
+    where they land. Custom destinations only need a single ``upload`` method.
+
+    Implementations consume :attr:`UploadContent.body` but do **not** own its
+    lifecycle -- the caller (the request) opens and closes the body stream.
+
+    Implementations should translate provider failures into
+    :class:`~remote_upload.exceptions.RetryableUploadError` (transient) or
+    :class:`~remote_upload.exceptions.TerminalUploadError` (permanent) so callers
+    can decide whether to retry.
+    """
+
+    def upload(self, content: UploadContent) -> UploadResult:
+        """Stream ``content`` into the destination and return the outcome."""
+        ...

remote_upload/targets/__init__.py

@@ -0,0 +1,3 @@
+"""Backend upload targets (HTTP, S3, Azure, GCS, SFTP, httpx)."""
+
+from __future__ import annotations

remote_upload/targets/azure.py

@@ -0,0 +1,103 @@
+"""Upload target that writes blobs to Azure Blob Storage."""
+
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+from azure.core.exceptions import AzureError, HttpResponseError, ServiceRequestError
+from azure.storage.blob import BlobClient, ContentSettings
+
+from ..content import UploadContent
+from ..exceptions import RetryableUploadError, TerminalUploadError
+from ..result import UploadResult
+
+logger = logging.getLogger(__name__)
+
+
+class AzureBlobTarget:
+    """:class:`~remote_upload.target.UploadTarget` for Azure Blob Storage.
+
+    Authenticates one of three ways: a full connection string, an endpoint URL
+    plus an optional SAS token, or a pre-built :class:`BlobClient` (handy for
+    reuse and tests). Uploads overwrite an existing blob (idempotent re-PUT of
+    the same key).
+    """
+
+    def __init__(
+        self,
+        *,
+        container: str,
+        blob: str,
+        connection_string: str | None = None,
+        endpoint: str | None = None,
+        sas_token: str | None = None,
+        client: BlobClient | None = None,
+    ) -> None:
+        if not container:
+            raise ValueError("AzureBlobTarget: container is required")
+        if not blob:
+            raise ValueError("AzureBlobTarget: blob is required")
+        if client is None and not connection_string and not endpoint:
+            raise ValueError("AzureBlobTarget: either connection_string, endpoint, or a client must be provided")
+        self._container = container
+        self._blob = blob
+        self._connection_string = connection_string
+        self._endpoint = endpoint
+        self._sas_token = sas_token
+        self._client = client
+
+    def _build_client(self) -> BlobClient:
+        """Build a :class:`BlobClient` from the configured connection string / endpoint."""
+        if self._connection_string:
+            return BlobClient.from_connection_string(
+                self._connection_string,
+                container_name=self._container,
+                blob_name=self._blob,
+            )
+        # __init__ guarantees endpoint is set when no connection string is given.
+        assert self._endpoint is not None
+        return BlobClient(
+            account_url=self._endpoint,
+            container_name=self._container,
+            blob_name=self._blob,
+            credential=self._sas_token,
+        )
+
+    def upload(self, content: UploadContent) -> UploadResult:
+        """Stream ``content`` into the blob, overwriting any existing object."""
+        logger.debug("[AzureBlobTarget] PUT %s/%s", self._container, self._blob)
+
+        owns_client = self._client is None
+        client = self._client or self._build_client()
+
+        kwargs: dict[str, Any] = {"data": content.body, "overwrite": True}
+        if content.content_length is not None:
+            kwargs["length"] = content.content_length
+        if content.content_type:
+            kwargs["content_settings"] = ContentSettings(content_type=content.content_type)
+        if content.metadata:
+            kwargs["metadata"] = dict(content.metadata)
+
+        try:
+            result = client.upload_blob(**kwargs)
+            return UploadResult(
+                key=self._blob,
+                location=client.url,
+                etag=result.get("etag"),
+                version_id=result.get("version_id"),
+                content_type=content.content_type,
+            )
+        except HttpResponseError as exc:
+            status = exc.status_code
+            msg = f"Azure upload {self._container}/{self._blob} failed (HTTP {status})"
+            if status is not None and 400 <= status < 500:
+                raise TerminalUploadError(msg) from exc
+            raise RetryableUploadError(msg) from exc
+        except (ServiceRequestError, AzureError) as exc:
+            raise RetryableUploadError(f"Azure upload {self._container}/{self._blob} failed") from exc
+        finally:
+            # Close only a client we built ourselves; an injected client is the
+            # caller's to manage.
+            if owns_client:
+                client.close()