cmem-plugin-s3 1.0.0__py3-none-any.whl

cmem_plugin_s3/__init__.py

@@ -0,0 +1 @@
+"""cmem-plugin-s3"""

cmem_plugin_s3/autocomplete.py

@@ -0,0 +1,130 @@
+"""Parameter types that ask the service what it holds"""
+
+from typing import Any, ClassVar
+
+from cmem_plugin_base.dataintegration.context import PluginContext
+from cmem_plugin_base.dataintegration.parameter.password import Password
+from cmem_plugin_base.dataintegration.types import Autocompletion, StringParameterType
+
+from cmem_plugin_s3.client import S3Client, S3Error, S3Settings
+
+CONNECTION_PARAMETERS: list[str] = [
+    "endpoint_url",
+    "region",
+    "credentials",
+    "access_key_id",
+    "secret_access_key",
+]
+"""The parameters an autocompletion needs before it can reach the service.
+
+Kept to the five that decide where to connect and as whom. The advanced
+connection parameters are deliberately left out: every entry here is a field
+the user has to fill before any list appears, and their defaults are right
+almost always.
+"""
+
+
+def _as_string(value: object) -> str:
+    """Read a parameter value that may arrive as a password."""
+    if isinstance(value, Password):
+        return str(value.decrypt())
+    return "" if value is None else str(value)
+
+
+def _client(values: list[Any]) -> S3Client:
+    """Build a client from the parameter values an autocompletion depends on.
+
+    The connection itself is opened by the first call that needs it, and an
+    endpoint or a credential the user has not finished typing raises there -
+    which is why every caller treats an S3Error as "nothing to offer yet".
+    """
+    endpoint_url, region, credentials, access_key_id, secret_access_key = (
+        _as_string(value) for value in values
+    )
+    settings = S3Settings(
+        endpoint_url=endpoint_url,
+        region=region,
+        credentials=credentials,
+        access_key_id=access_key_id,
+        secret_access_key=secret_access_key,
+    )
+    return S3Client(settings)
+
+
+def _matching(completions: list[Autocompletion], query_terms: list[str]) -> list[Autocompletion]:
+    """Keep the completions that match every term the user typed."""
+    if not query_terms:
+        return completions
+    return [
+        completion
+        for completion in completions
+        if all(
+            term.lower() in (completion.label or completion.value).lower() for term in query_terms
+        )
+    ]
+
+
+class BucketParameterType(StringParameterType):
+    """Offers the buckets the credentials may see
+
+    Any bucket name may still be typed by hand. Listing buckets needs a
+    permission a key scoped to a single bucket does not have, and that key is
+    the well configured case rather than the broken one.
+    """
+
+    allow_only_autocompleted_values: bool = False
+    autocomplete_value_with_labels: bool = False
+    autocompletion_depends_on_parameters: ClassVar[list[str]] = CONNECTION_PARAMETERS
+
+    def autocomplete(
+        self,
+        query_terms: list[str],
+        depend_on_parameter_values: list[Any],
+        context: PluginContext,
+    ) -> list[Autocompletion]:
+        """Return the buckets matching all query terms."""
+        _ = context
+        try:
+            buckets = _client(depend_on_parameter_values).buckets()
+        except S3Error:
+            return []
+        return _matching([Autocompletion(value=name, label=name) for name in buckets], query_terms)
+
+
+class PrefixParameterType(StringParameterType):
+    """Offers the key prefixes of the selected bucket, one level at a time
+
+    What is offered are the prefixes directly below what is typed, so a prefix
+    is assembled by selecting its parts in turn rather than by searching the
+    whole key space - a bucket holds far too many keys for the latter.
+    """
+
+    allow_only_autocompleted_values: bool = False
+    autocomplete_value_with_labels: bool = False
+    autocompletion_depends_on_parameters: ClassVar[list[str]] = [
+        *CONNECTION_PARAMETERS,
+        "bucket",
+    ]
+
+    def autocomplete(
+        self,
+        query_terms: list[str],
+        depend_on_parameter_values: list[Any],
+        context: PluginContext,
+    ) -> list[Autocompletion]:
+        """Return the prefixes below the one that is typed."""
+        _ = context
+        *connection, bucket = depend_on_parameter_values
+        bucket = _as_string(bucket).strip()
+        if not bucket:
+            return []
+        typed = " ".join(query_terms).strip()
+        below = typed[: typed.rindex("/") + 1] if "/" in typed else ""
+        try:
+            prefixes = _client(connection).prefixes(bucket, below)
+        except S3Error:
+            return []
+        completions = [Autocompletion(value=prefix, label=prefix) for prefix in prefixes]
+        if not typed:
+            return completions
+        return [completion for completion in completions if completion.value.startswith(typed)]

cmem_plugin_s3/client.py

@@ -0,0 +1,366 @@
+"""Talking to one S3 endpoint"""
+
+import re
+import shutil
+from collections.abc import Iterator
+from dataclasses import dataclass
+from datetime import datetime
+from pathlib import Path, PurePosixPath
+from typing import IO, TYPE_CHECKING, Any, Final, Literal
+
+import boto3
+from botocore.config import Config
+from botocore.exceptions import BotoCoreError, ClientError
+
+if TYPE_CHECKING:  # pragma: no cover - the stubs are a development dependency
+    from types_boto3_s3.client import S3Client as BotoS3
+
+AUTO: Final = "auto"
+PATH: Final = "path"
+VIRTUAL: Final = "virtual"
+EXPLICIT: Final = "explicit"
+
+CHUNK_SIZE = 1024 * 1024
+"""How much of an object is held in memory while it is written to disk."""
+
+DENIED_CODES = frozenset({"AccessDenied", "AllAccessDisabled", "InvalidAccessKeyId"})
+"""What a service answers when the credentials may not do what was asked."""
+
+NOT_FOUND = 404
+PRECONDITION_FAILED = 412
+
+
+class S3Error(Exception):
+    """Something went wrong while talking to the service"""
+
+
+@dataclass(frozen=True)
+class S3Settings:
+    """Everything needed to reach one endpoint as one identity"""
+
+    endpoint_url: str = ""
+    region: str = "us-east-1"
+    credentials: str = EXPLICIT
+    access_key_id: str = ""
+    secret_access_key: str = ""
+    session_token: str = ""
+    addressing_style: str = AUTO
+    ssl_verify: bool = True
+
+    @property
+    def resolved_addressing_style(self) -> Literal["path", "virtual"]:
+        """Decide how a bucket is named in a request.
+
+        An endpoint URL means an implementation other than Amazon S3, and those
+        are commonly deployed without the wildcard DNS entry virtual-hosted
+        addressing needs - so the automatic choice is path style there and
+        virtual-hosted style against Amazon S3.
+        """
+        if self.addressing_style == PATH:
+            return PATH
+        if self.addressing_style == VIRTUAL:
+            return VIRTUAL
+        return PATH if self.endpoint_url.strip() else VIRTUAL
+
+
+@dataclass(frozen=True)
+class S3Object:
+    """One object of a bucket, as the service describes it"""
+
+    bucket: str
+    key: str
+    size: int = 0
+    etag: str = ""
+    last_modified: str = ""
+    storage_class: str = ""
+
+    @property
+    def uri(self) -> str:
+        """Name the object the way a user writes it."""
+        return f"s3://{self.bucket}/{self.key}"
+
+    @property
+    def name(self) -> str:
+        """Reduce the key to a plain file name
+
+        A key is free text that whoever wrote the object decided, so it may end
+        in a separator, may consist of nothing but dots, or may be empty. None
+        of that is allowed to decide where a workflow node writes, and an empty
+        name would write onto the directory itself.
+        """
+        name = PurePosixPath(self.key.replace("\\", "/")).name
+        name = name.replace("\0", "").strip().strip(".")
+        return name or "object"
+
+    def values(self) -> list[list[str]]:
+        """Lay the object out in the order of the object schema's paths."""
+        return [
+            [self.bucket],
+            [self.key],
+            [self.name],
+            [str(self.size)],
+            [self.etag],
+            [self.last_modified],
+            [self.storage_class],
+        ]
+
+
+def quoted_etag(etag: str) -> str:
+    """Put an entity tag into the quoted form the service compares against.
+
+    A listing reports the tag quoted and a user who copies it into a transform
+    may well drop the quotes, so both forms have to be accepted here.
+    """
+    stripped = etag.strip()
+    if not stripped:
+        return ""
+    if stripped.startswith('"') and stripped.endswith('"'):
+        return stripped
+    return f'"{stripped}"'
+
+
+def _unquoted_etag(etag: str | None) -> str:
+    """Keep an entity tag without the quotes the service wraps it in."""
+    return (etag or "").strip().strip('"')
+
+
+def _is_denied(error: ClientError) -> bool:
+    """Check whether a service refused the call for lack of permission."""
+    return str(error.response.get("Error", {}).get("Code", "")) in DENIED_CODES
+
+
+class S3Client:
+    """Talks to one S3 endpoint as one identity"""
+
+    def __init__(self, settings: S3Settings) -> None:
+        self.settings = settings
+        self._client: BotoS3 | None = None
+        self._session: boto3.Session | None = None
+
+    @property
+    def client(self) -> "BotoS3":
+        """Build the connection, once per task run."""
+        if self._client is None:
+            self._client = self._build()
+        return self._client
+
+    def _build(self) -> "BotoS3":
+        """Configure boto3 from the settings.
+
+        The credentials go on the session rather than on the client, so that
+        asking the session who it signs as answers for the explicit key as well
+        as for a key the chain resolved.
+        """
+        settings = self.settings
+        session_arguments: dict[str, Any] = {}
+        if settings.region.strip():
+            session_arguments["region_name"] = settings.region.strip()
+        if settings.credentials == EXPLICIT:
+            if not settings.access_key_id.strip() or not settings.secret_access_key:
+                raise S3Error(
+                    "An access key ID and a secret access key are needed, or the credentials "
+                    "have to be taken from the node identity."
+                )
+            session_arguments["aws_access_key_id"] = settings.access_key_id.strip()
+            session_arguments["aws_secret_access_key"] = settings.secret_access_key
+            if settings.session_token:
+                session_arguments["aws_session_token"] = settings.session_token
+        client_arguments: dict[str, Any] = {
+            "config": Config(
+                s3={"addressing_style": settings.resolved_addressing_style},
+                signature_version="s3v4",
+            ),
+            "verify": settings.ssl_verify,
+        }
+        if settings.endpoint_url.strip():
+            client_arguments["endpoint_url"] = settings.endpoint_url.strip()
+        try:
+            self._session = boto3.Session(**session_arguments)
+            return self._session.client("s3", **client_arguments)
+        except (BotoCoreError, ValueError) as error:
+            raise S3Error(f"The connection could not be configured: {error}") from error
+
+    def buckets(self) -> list[str]:
+        """List the buckets the credentials may see.
+
+        Returns an empty list when listing buckets is not permitted, which is
+        the common case for a key scoped to a single bucket.
+        """
+        try:
+            response = self.client.list_buckets()
+        except ClientError as error:
+            if _is_denied(error):
+                return []
+            raise S3Error(f"The buckets could not be listed: {error}") from error
+        except BotoCoreError as error:
+            raise S3Error(f"The buckets could not be listed: {error}") from error
+        return sorted(str(bucket["Name"]) for bucket in response.get("Buckets", []))
+
+    def prefixes(self, bucket: str, prefix: str) -> list[str]:
+        """List the key prefixes directly below a prefix."""
+        try:
+            response = self.client.list_objects_v2(
+                Bucket=bucket, Prefix=prefix, Delimiter="/", MaxKeys=1000
+            )
+        except (ClientError, BotoCoreError) as error:
+            raise S3Error(f"'{bucket}' could not be listed: {error}") from error
+        return sorted(
+            str(entry["Prefix"])
+            for entry in response.get("CommonPrefixes", [])
+            if entry.get("Prefix")
+        )
+
+    def objects(self, bucket: str, prefix: str, top_level_only: bool) -> Iterator[S3Object]:
+        """Walk the objects below a prefix, one page of the listing at a time.
+
+        A key ending in a separator is a folder marker rather than content -
+        some tools write one so that a console shows an empty folder - and is
+        left out.
+        """
+        arguments: dict[str, Any] = {"Bucket": bucket, "Prefix": prefix}
+        if top_level_only:
+            arguments["Delimiter"] = "/"
+        try:
+            for page in self.client.get_paginator("list_objects_v2").paginate(**arguments):
+                for entry in page.get("Contents", []):
+                    key = str(entry["Key"])
+                    if key.endswith("/"):
+                        continue
+                    yield S3Object(
+                        bucket=bucket,
+                        key=key,
+                        size=int(entry.get("Size", 0)),
+                        etag=_unquoted_etag(entry.get("ETag")),
+                        last_modified=self._timestamp(entry.get("LastModified")),
+                        storage_class=str(entry.get("StorageClass", "STANDARD")),
+                    )
+        except (ClientError, BotoCoreError) as error:
+            raise S3Error(f"'{bucket}' could not be listed: {error}") from error
+
+    @staticmethod
+    def _timestamp(value: datetime | None) -> str:
+        """Write a modification time down the way a graph can compare it."""
+        return value.isoformat() if value is not None else ""
+
+    def head(self, bucket: str, key: str) -> S3Object | None:
+        """Describe one object, or report that the bucket does not hold it."""
+        try:
+            response = self.client.head_object(Bucket=bucket, Key=key)
+        except ClientError as error:
+            status = int(error.response.get("ResponseMetadata", {}).get("HTTPStatusCode", 0))
+            if status == NOT_FOUND:
+                return None
+            raise S3Error(f"'{key}' could not be read: {error}") from error
+        except BotoCoreError as error:
+            raise S3Error(f"'{key}' could not be read: {error}") from error
+        return S3Object(
+            bucket=bucket,
+            key=key,
+            size=int(response.get("ContentLength", 0)),
+            etag=_unquoted_etag(response.get("ETag")),
+            last_modified=self._timestamp(response.get("LastModified")),
+            storage_class=str(response.get("StorageClass", "STANDARD")),
+        )
+
+    def download(self, obj: S3Object, directory: Path) -> tuple[Path, str | None]:
+        """Fetch one object into a directory, and report its media type.
+
+        The entity tag the listing reported is sent along, so an object that was
+        replaced between the listing and this call fails rather than quietly
+        delivering different bytes than the workflow selected.
+        """
+        arguments: dict[str, Any] = {"Bucket": obj.bucket, "Key": obj.key}
+        if obj.etag:
+            arguments["IfMatch"] = quoted_etag(obj.etag)
+        try:
+            response = self.client.get_object(**arguments)
+        except ClientError as error:
+            status = int(error.response.get("ResponseMetadata", {}).get("HTTPStatusCode", 0))
+            if status == PRECONDITION_FAILED:
+                raise S3Error(
+                    f"'{obj.uri}' was replaced after it was listed, so it was not downloaded."
+                ) from error
+            raise S3Error(f"'{obj.uri}' could not be downloaded: {error}") from error
+        except BotoCoreError as error:
+            raise S3Error(f"'{obj.uri}' could not be downloaded: {error}") from error
+        target = directory / obj.name
+        with response["Body"] as body, target.open("wb") as sink:
+            shutil.copyfileobj(body, sink, CHUNK_SIZE)
+        content_type = response.get("ContentType")
+        return target, str(content_type) if content_type else None
+
+    def upload(self, bucket: str, key: str, source: IO[bytes]) -> S3Object:
+        """Write one object, in as many parts as its size needs."""
+        try:
+            self.client.upload_fileobj(source, bucket, key)
+        except (ClientError, BotoCoreError) as error:
+            raise S3Error(f"'s3://{bucket}/{key}' could not be uploaded: {error}") from error
+        written = self.head(bucket, key)
+        return written or S3Object(bucket=bucket, key=key)
+
+    def report(self, bucket: str) -> str:
+        """Say how far an attempt to reach and use the service gets.
+
+        Written for the action on the task form, where every step that did work
+        narrows down the one that did not.
+        """
+        settings = self.settings
+        lines = [
+            f"* Endpoint: `{settings.endpoint_url or 'https://s3.<region>.amazonaws.com'}`",
+            f"* Region: `{settings.region or '(none)'}`",
+            f"* Addressing style: `{settings.resolved_addressing_style}`",
+            f"* Credentials: `{settings.credentials}`",
+        ]
+        try:
+            identity = self._signer()
+        except S3Error as error:
+            return "\n".join([*lines, "", f"**Not configured.** {error}"])
+        lines.append(f"* Signing as: {identity}")
+        buckets = []
+        try:
+            buckets = self.buckets()
+        except S3Error as error:
+            return "\n".join([*lines, "", f"**The service could not be reached.** {error}"])
+        lines.append(
+            f"* Buckets visible: {len(buckets)}"
+            if buckets
+            else "* Buckets visible: none, which is normal for a key scoped to one bucket"
+        )
+        if not bucket.strip():
+            return "\n".join([*lines, "", "**The service answered.** No bucket is configured."])
+        return "\n".join([*lines, "", self._bucket_report(bucket.strip())])
+
+    def _bucket_report(self, bucket: str) -> str:
+        """Say whether the configured bucket can be reached and read."""
+        try:
+            self.client.head_bucket(Bucket=bucket)
+        except (ClientError, BotoCoreError) as error:
+            return f"**`{bucket}` could not be reached.** {error}"
+        try:
+            page = self.client.list_objects_v2(Bucket=bucket, MaxKeys=1)
+        except (ClientError, BotoCoreError) as error:
+            return f"**`{bucket}` exists but could not be listed.** {error}"
+        held = "holds at least one object" if page.get("Contents") else "is empty"
+        return f"**`{bucket}` can be reached and listed**, and {held}."
+
+    def _signer(self) -> str:
+        """Build the connection and name the access key it signs with.
+
+        Read from the session rather than from the client, so that the ambient
+        credential chain is resolved the same way a request would resolve it.
+        """
+        _ = self.client
+        if self._session is None:
+            return "nothing, the connection was not built"
+        credentials = self._session.get_credentials()
+        if credentials is None:
+            return "nothing, no credentials were found"
+        return f"`{credentials.get_frozen_credentials().access_key}`"
+
+
+def compiled(regex: str) -> re.Pattern[str]:
+    """Compile a key expression, or one that matches every key."""
+    try:
+        return re.compile(regex) if regex.strip() else re.compile("")
+    except re.error as error:
+        raise S3Error(f"'{regex}' is not a valid regular expression: {error}") from error