h2o-drive 0.0.1__tar.gz

h2o_drive-0.0.1/PKG-INFO

@@ -0,0 +1,113 @@
+Metadata-Version: 2.1
+Name: h2o-drive
+Version: 0.0.1
+Summary: H2O Drive Python Client
+Author: H2O.ai
+Author-email: support@h2o.ai
+Requires-Python: >=3.7,<4.0
+Classifier: Intended Audience :: Developers
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.7
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Topic :: Internet
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Dist: aioboto3 (>=9.3.1,<10.0.0)
+Requires-Dist: aiobotocore (>=2.1.0,<3.0.0)
+Requires-Dist: botocore (==1.23.24)
+Requires-Dist: h2o-authn[discovery] (>=1.1.0,<2.0.0)
+Requires-Dist: h2o-cloud-discovery (>=1.1.0,<2.0.0)
+Requires-Dist: typing-extensions (>=3.7.4)
+Description-Content-Type: text/markdown
+
+# h2o-drive
+
+H2O Drive Python Client.
+
+## Installation
+
+```sh
+pip install h2o-drive
+```
+
+## Usage
+
+### Connecting to H2O Drive
+
+```py
+import h2o_drive
+
+drive = await h2o_drive.Drive()
+```
+
+When used within the [H2O AI Cloud](https://h2o.ai/platform/ai-cloud/) environment or locally with the [H2O AI Cloud CLI](https://docs.h2o.ai/h2o-ai-cloud/developerguide/cli) configured, no further configuration is needed.
+
+`h2o_drive.Drive()` can be configured via its optional parameters:
+
+- `token`: Token or token provider to use. If not specified, it's determined automatically.
+- `endpoint_url`: URL of the Drive endpoint. If not specified, it's determined automatically.
+- `sts_endpoint_url`: URL of the STS endpoint. If not specified, it's determined automatically.
+- `environment`: URL of the H2O.ai cloud environment. Used only when determining a missing token or URL.
+- `config_path`: Path to the H2O.ai CLI config. Used only when determining a missing token or URL.
+- `session_ttl_seconds`: The duration, in seconds, of the role session.
+- `read_timeout_seconds`: The time in seconds until a timeout exception is thrown when reading from Drive.
+
+### Object Operations
+
+To perform object operations, first navigate to the user's personal drive bucket:
+
+```py
+my_bucket = drive.my_bucket()
+```
+
+Continue operating at the root of the user's bucket (generally not needed) or navigate into a particular space (a.k.a. a directory path) with one of the following.  The `home` space is a good place to start.
+
+```py
+my_home_space = my_bucket.home()
+# or
+my_other_space = my_bucket.workspace("my_other_space")
+```
+
+Once in a space within a user's bucket (or the root of the bucket), the following operations are possible:
+
+- `upload_file` uploads a local file to the user's Drive. It takes two arguments:
+    - `file_name`: The file to upload.
+    - `object_name`: The name to give to the uploaded file once it becomes an object in our Drive bucket.
+
+- `list_objects` lists objects in the user's drive. It takes one optional argument:
+    - `prefix`: When set, only objects whose names start with the specified prefix are returned.
+
+- `download_file` downloads an object as a local file. It takes two arguments:
+    - `object_name`: The name of the object (including prefix) to download.
+    - `file_name`: Path to where the object should be saved as a local file.
+
+- `delete_object` Deletes an object from the user's drive. It takes a single argument:
+  - `object_name`: The name of the object (including prefix) to delete.
+
+- `generate_presigned_url` generates a presigned URL through which an object in the drive can be accessed. It takes two arguments:
+  - `object_name`: The name of the object (including prefix) to generate a presigned URL for.
+  - `ttl_seconds`: (Optional) How long, in seconds, the URL should be good for.
+
+## Examples
+
+### Example: Connect to the `home` space of a user's drive bucket, write a local file, and then list the contents
+
+```py
+import h2o_drive
+
+drive = await h2o_drive.Drive()
+
+my_home_space = drive.my_bucket().home()
+
+with open("test-file.txt", "w") as f:
+    f.write("Hello, world!")
+
+await my_home_space.upload_file("test-file.txt", "uploaded-object-name.txt")
+
+objects = await my_home_space.list_objects()
+print(objects)
+```
+

h2o_drive-0.0.1/PUBLIC_README.md

@@ -0,0 +1,87 @@
+# h2o-drive
+
+H2O Drive Python Client.
+
+## Installation
+
+```sh
+pip install h2o-drive
+```
+
+## Usage
+
+### Connecting to H2O Drive
+
+```py
+import h2o_drive
+
+drive = await h2o_drive.Drive()
+```
+
+When used within the [H2O AI Cloud](https://h2o.ai/platform/ai-cloud/) environment or locally with the [H2O AI Cloud CLI](https://docs.h2o.ai/h2o-ai-cloud/developerguide/cli) configured, no further configuration is needed.
+
+`h2o_drive.Drive()` can be configured via its optional parameters:
+
+- `token`: Token or token provider to use. If not specified, it's determined automatically.
+- `endpoint_url`: URL of the Drive endpoint. If not specified, it's determined automatically.
+- `sts_endpoint_url`: URL of the STS endpoint. If not specified, it's determined automatically.
+- `environment`: URL of the H2O.ai cloud environment. Used only when determining a missing token or URL.
+- `config_path`: Path to the H2O.ai CLI config. Used only when determining a missing token or URL.
+- `session_ttl_seconds`: The duration, in seconds, of the role session.
+- `read_timeout_seconds`: The time in seconds until a timeout exception is thrown when reading from Drive.
+
+### Object Operations
+
+To perform object operations, first navigate to the user's personal drive bucket:
+
+```py
+my_bucket = drive.my_bucket()
+```
+
+Continue operating at the root of the user's bucket (generally not needed) or navigate into a particular space (a.k.a. a directory path) with one of the following.  The `home` space is a good place to start.
+
+```py
+my_home_space = my_bucket.home()
+# or
+my_other_space = my_bucket.workspace("my_other_space")
+```
+
+Once in a space within a user's bucket (or the root of the bucket), the following operations are possible:
+
+- `upload_file` uploads a local file to the user's Drive. It takes two arguments:
+    - `file_name`: The file to upload.
+    - `object_name`: The name to give to the uploaded file once it becomes an object in our Drive bucket.
+
+- `list_objects` lists objects in the user's drive. It takes one optional argument:
+    - `prefix`: When set, only objects whose names start with the specified prefix are returned.
+
+- `download_file` downloads an object as a local file. It takes two arguments:
+    - `object_name`: The name of the object (including prefix) to download.
+    - `file_name`: Path to where the object should be saved as a local file.
+
+- `delete_object` Deletes an object from the user's drive. It takes a single argument:
+  - `object_name`: The name of the object (including prefix) to delete.
+
+- `generate_presigned_url` generates a presigned URL through which an object in the drive can be accessed. It takes two arguments:
+  - `object_name`: The name of the object (including prefix) to generate a presigned URL for.
+  - `ttl_seconds`: (Optional) How long, in seconds, the URL should be good for.
+
+## Examples
+
+### Example: Connect to the `home` space of a user's drive bucket, write a local file, and then list the contents
+
+```py
+import h2o_drive
+
+drive = await h2o_drive.Drive()
+
+my_home_space = drive.my_bucket().home()
+
+with open("test-file.txt", "w") as f:
+    f.write("Hello, world!")
+
+await my_home_space.upload_file("test-file.txt", "uploaded-object-name.txt")
+
+objects = await my_home_space.list_objects()
+print(objects)
+```

h2o_drive-0.0.1/h2o_drive/__init__.py

@@ -0,0 +1,245 @@
+import os
+from typing import Optional
+from typing import Union
+
+import h2o_authn.discovery
+import h2o_discovery
+from h2o_drive import auth
+from h2o_drive import core
+
+CLOUD_DISCOVERY_SERVICE_NAME = "drive"
+
+Workspace = core.Workspace
+MY_BUCKET = core.MY_BUCKET
+
+
+async def MyHome(
+    token: Union[auth.TokenProvider, str],
+    *,
+    endpoint_url: Optional[str] = None,
+    sts_endpoint_url: Optional[str] = None,
+    environment: Optional[str] = None,
+    config_path: Optional[str] = None,
+    session_ttl_seconds: int = 3600,
+    read_timeout_seconds: int = 60,
+) -> core.PrefixClient:
+    """Returns new client connected to the user's personal bucket and the HOME
+    workspace.
+
+    When the personal bucket does not exist, it's created.
+
+    When not specified, certain parameters are determined automatically by
+    either environment variables or via the H2O Cloud Discovery Client.  While
+    the Cloud Discovery Client may apply multiple strategies, the following
+    rules generally apply:
+    - environment, if specified, will be used instead of searching for that
+        value in config_path.
+    - The H2O Cloud Discovery Client will fall back to common H2O environment
+        variables and configuration paths, but will only supersede values
+        specified here if a more specific environment variable,
+        H2O_CLOUD_DISCOVERY, is found. This value is often set in H2O cloud
+        applications.
+
+    Args:
+        token: Token or token provider to use. If not specified, it's determined
+            automatically.
+        endpoint_url: URL of the Drive endpoint. If not specified, it's
+            determined automatically.
+        sts_endpoint_url: URL of the STS endpoint. If not specified, it's
+            determined automatically.
+        environment: URL of the H2O.ai cloud environment. Used only when
+            determining a missing token or URL.
+        config_path: Path to the H2O.ai CLI config. Used only when determining a
+            missing token or URL.
+        session_ttl_seconds: The duration, in seconds, of the role session.
+        read_timeout_seconds: The time in seconds until a timeout exception is
+            thrown when reading from Drive.
+
+    Returns:
+        A Drive client connected to the user's personal bucket and the HOME
+            workspace.
+    """
+
+    my_bucket = await MyBucket(
+        token=token,
+        endpoint_url=endpoint_url,
+        sts_endpoint_url=sts_endpoint_url,
+        environment=environment,
+        config_path=config_path,
+        session_ttl_seconds=session_ttl_seconds,
+        read_timeout_seconds=read_timeout_seconds,
+    )
+    return my_bucket.home()
+
+
+async def MyBucket(
+    token: Union[auth.TokenProvider, str],
+    *,
+    endpoint_url: Optional[str] = None,
+    sts_endpoint_url: Optional[str] = None,
+    environment: Optional[str] = None,
+    config_path: Optional[str] = None,
+    session_ttl_seconds: int = 3600,
+    read_timeout_seconds: int = 60,
+) -> core.Bucket:
+    """Returns new client connected to the user's personal bucket.
+
+    When the personal bucket does not exist, it's created.
+
+    When not specified, certain parameters are determined automatically by
+    either environment variables or via the H2O Cloud Discovery Client.  While
+    the Cloud Discovery Client may apply multiple strategies, the following
+    rules generally apply:
+    - environment, if specified, will be used instead of searching for that
+        value in config_path.
+    - The H2O Cloud Discovery Client will fall back to common H2O environment
+        variables and configuration paths, but will only supersede values
+        specified here if a more specific environment variable,
+        H2O_CLOUD_DISCOVERY, is found. This value is often set in H2O cloud
+        applications.
+
+    Args:
+        token: Token or token provider to use. If not specified, it's determined
+            automatically.
+        endpoint_url: URL of the Drive endpoint. If not specified, it's
+            determined automatically.
+        sts_endpoint_url: URL of the STS endpoint. If not specified, it's
+            determined automatically.
+        environment: URL of the H2O.ai cloud environment. Used only when
+            determining a missing token or URL.
+        config_path: Path to the H2O.ai CLI config. Used only when determining a
+            missing token or URL.
+        session_ttl_seconds: The duration, in seconds, of the role session.
+        read_timeout_seconds: The time in seconds until a timeout exception is
+            thrown when reading from Drive.
+
+    Returns:
+        A Drive client connected to the user's personal bucket.
+    """
+    client = await Drive(
+        token=token,
+        endpoint_url=endpoint_url,
+        sts_endpoint_url=sts_endpoint_url,
+        environment=environment,
+        config_path=config_path,
+        session_ttl_seconds=session_ttl_seconds,
+        read_timeout_seconds=read_timeout_seconds,
+    )
+    bucket = client.my_bucket()
+    await bucket.ensure_created()
+    return bucket
+
+
+async def Drive(
+    token: Union[auth.TokenProvider, str, None] = None,
+    *,
+    endpoint_url: Optional[str] = None,
+    sts_endpoint_url: Optional[str] = None,
+    environment: Optional[str] = None,
+    config_path: Optional[str] = None,
+    session_ttl_seconds: int = 3600,
+    read_timeout_seconds: int = 60,
+) -> core.Client:
+    """Returns a new instance of the Drive client.
+
+    When not specified, certain parameters are determined automatically by
+    either environment variables or via the H2O Cloud Discovery Client.  While
+    the Cloud Discovery Client may apply multiple strategies, the following
+    rules generally apply:
+    - environment, if specified, will be used instead of searching for that
+        value in config_path.
+    - The H2O Cloud Discovery Client will fall back to common H2O environment
+        variables and configuration paths, but will only supersede values
+        specified here if a more specific environment variable,
+        H2O_CLOUD_DISCOVERY, is found. This value is often set in H2O cloud
+        applications.
+
+    Args:
+        token: Token or token provider to use. If not specified, it's determined
+            automatically.
+        endpoint_url: URL of the Drive endpoint. If not specified, it's
+            determined automatically.
+        sts_endpoint_url: URL of the STS endpoint. If not specified, it's
+            determined automatically.
+        environment: URL of the H2O.ai cloud environment. Used only when
+            determining a missing token or URL.
+        config_path: Path to the H2O.ai CLI config. Used only when determining a
+            missing token or URL.
+        session_ttl_seconds: The duration, in seconds, of the role session.
+        read_timeout_seconds: The time in seconds until a timeout exception is
+            thrown when reading from Drive.
+
+    Returns:
+        A Drive client.
+    """
+    if token is None or endpoint_url is None:
+        discovery = h2o_discovery.discover(
+            environment=environment, config_path=config_path
+        )
+
+    if token is None:
+        token_provider = discover_token_provider(discovery)
+    elif isinstance(token, auth.TokenProvider):
+        token_provider = token
+    else:
+        token_provider = auth.StaticTokenProvider(token)
+
+    if endpoint_url is None:
+        endpoint_url = os.getenv("H2O_DRIVE_ENDPOINT")
+    if endpoint_url is None:
+        service = discovery.services.get(CLOUD_DISCOVERY_SERVICE_NAME)
+        if service is not None:
+            endpoint_url = service.uri
+    if endpoint_url is None:
+        raise LookupError(
+            "Failed to automatically determine the endpoint URL. "
+            "Drive entry missing from Cloud Discovery service."
+        )
+        endpoint_url = service.uri
+
+    if sts_endpoint_url is None:
+        sts_endpoint_url = os.getenv("H2O_DRIVE_STS_ENDPOINT")
+    if sts_endpoint_url is None:
+        sts_endpoint_url = f'{endpoint_url.rstrip("/")}/sts/'
+
+    return await connect(
+        token_provider=token_provider,
+        sts_endpoint_url=sts_endpoint_url,
+        endpoint_url=endpoint_url,
+        session_ttl_seconds=session_ttl_seconds,
+        read_timeout_seconds=read_timeout_seconds,
+    )
+
+
+async def connect(
+    token_provider: auth.TokenProvider,
+    *,
+    endpoint_url: str,
+    sts_endpoint_url: str,
+    session_ttl_seconds: int = 3600,
+    read_timeout_seconds: int = 60,
+) -> core.Client:
+    """Returns a new instance of the Client with session created against the
+    token returned by the token provider.
+    """
+
+    session_provider = auth.RefreshingWebIdentitySessionProvider(
+        token_provider=token_provider,
+        sts_endpoint_url=sts_endpoint_url,
+        session_ttl_seconds=session_ttl_seconds,
+        read_timeout_seconds=read_timeout_seconds,
+    )
+    return core.Client(session=session_provider, endpoint_url=endpoint_url)
+
+
+def discover_token_provider(discovery: h2o_discovery.Discovery) -> auth.TokenProvider:
+    """Returns a new instance of an async TokenProvider configured from the
+    given Discovery object.
+    """
+    try:
+        return h2o_authn.discovery.create_async(discovery)
+    except KeyError:
+        raise LookupError(
+            "Failed to automatically determine the token provider from "
+            "information available in the environment."
+        )

h2o_drive-0.0.1/h2o_drive/auth.py

@@ -0,0 +1,123 @@
+import datetime
+from typing import Optional
+
+import aioboto3
+import aiobotocore.config
+import aiobotocore.session
+import typing_extensions
+from typing_extensions import Protocol
+
+
+DEFAULT_SESSION_EXPIRATION_THRESHOLD = datetime.timedelta(minutes=3)
+
+
+@typing_extensions.runtime_checkable
+class TokenProvider(Protocol):
+    """Defines the interface of the token provider.
+
+    Token provider is used to obtain access token that is exchanged for the session
+    token.
+    """
+
+    async def __call__(self) -> str:
+        """Returns fresh access token for the access to H2O Drive server."""
+
+
+class StaticTokenProvider:
+    """Implements TokenProvider interface and always returns static token passed."""
+
+    def __init__(self, token: str) -> None:
+        self._token = token
+
+    async def __call__(self) -> str:
+        return self._token
+
+
+@typing_extensions.runtime_checkable
+class SessionProvider(Protocol):
+    """Defines the interface of the session provider.
+
+    Session provider is used by the core components to get access to the H2O Drive
+    server.
+    """
+
+    async def __call__(self) -> aioboto3.Session:
+        pass
+
+
+class StaticSessionProvider:
+    """Implements SessionProvider interface and always returns the session passed
+    to the constructor.
+    """
+
+    def __init__(self, session: aioboto3.Session) -> None:
+        self._session = session
+
+    async def __call__(self) -> aioboto3.Session:
+        return self._session
+
+
+class RefreshingWebIdentitySessionProvider:
+    """Uses token provider passed to the constructor to obtain fresh session credentials
+    when needed.
+    """
+
+    def __init__(
+        self,
+        token_provider: TokenProvider,
+        sts_endpoint_url: str,
+        *,
+        session_ttl_seconds: int = 3600,
+        expiration_threshold: datetime.timedelta = DEFAULT_SESSION_EXPIRATION_THRESHOLD,
+        read_timeout_seconds: int = 60,
+    ) -> None:
+        self._token_provider = token_provider
+        self._sts_endpoint_url = sts_endpoint_url
+        self._expiration_threshold = expiration_threshold
+        self._session_ttl_seconds = session_ttl_seconds
+        self._read_timeout_seconds = read_timeout_seconds
+
+        self._session: Optional[aioboto3.Session] = None
+        self._credentials_expiration: Optional[datetime.datetime] = None
+
+    async def __call__(self) -> aioboto3.Session:
+        if self.refresh_required():
+            await self._do_refresh()
+        return self._session
+
+    def refresh_required(self) -> bool:
+        """Returns True when there's no access token set or the current one requires
+        refresh.
+        """
+        if self._session is None or self._credentials_expiration is None:
+            return True
+
+        now = datetime.datetime.now(datetime.timezone.utc)
+        return self._credentials_expiration <= (now + self._expiration_threshold)
+
+    async def _do_refresh(self) -> aioboto3.Session:
+        credentials = await self._get_credentials()
+        self._credentials_expiration = credentials.get("Expiration")
+        self._session = aioboto3.Session(
+            aws_access_key_id=credentials.get("AccessKeyId"),
+            aws_secret_access_key=credentials.get("SecretAccessKey"),
+            aws_session_token=credentials.get("SessionToken"),
+        )
+
+    async def _get_credentials(self):
+        boto_session = aiobotocore.session.get_session()
+        async with boto_session.create_client(
+            "sts",
+            endpoint_url=self._sts_endpoint_url,
+            config=aiobotocore.config.AioConfig(
+                read_timeout=self._read_timeout_seconds
+            ),
+        ) as sts:
+            assume_role = await sts.assume_role_with_web_identity(
+                # RoleArn is an arbitrary string from 20 to 2048 characters long.
+                RoleArn="h2odriveh2odriveh2odrive",
+                RoleSessionName="h2odrive",
+                WebIdentityToken=await self._token_provider(),
+                DurationSeconds=self._session_ttl_seconds,
+            )
+        return assume_role.get("Credentials", dict())

h2o_drive-0.0.1/h2o_drive/core.py

@@ -0,0 +1,315 @@
+import contextlib
+import datetime
+import enum
+import os
+import pathlib
+from typing import List
+from typing import NamedTuple
+from typing import Optional
+
+import boto3.s3.transfer
+import botocore.exceptions
+
+from h2o_drive import auth
+
+MY_BUCKET = "_my"
+_HOME_PREFIX = "home/"
+_DEFAULT_PRESIGNED_URL_TTL_SECONDS = 24 * 60 * 60  # 1 day.
+
+
+class Workspace(enum.Enum):
+    """Defines sopported predefined workspaces."""
+
+    HOME = enum.auto()
+    APP = enum.auto()
+    APP_VERSION = enum.auto()
+    APP_INSTANCE = enum.auto()
+
+
+class PresignedURL(str):
+    """Defines presigned URL as it's own type so it can be extended for integration with
+    other service clients (e.g. H2O-3)
+    """
+
+
+class ObjectSummary(NamedTuple):
+    """Represents basic information about object."""
+
+    key: str
+    size: int
+    last_modified: datetime.datetime
+
+
+class PrefixClient:
+    """Implements standard object operations but all keys are automatically prefixed
+    with configured prefix.
+
+    Simulates behavior of the bucket within the bucket.
+    """
+
+    def __init__(self, *, bucket: "Bucket", prefix: str) -> None:
+        self._bucket = bucket
+        self._prefix = prefix
+        self._prefix_length = len(prefix)
+
+    async def upload_file(self, file_name: os.PathLike, object_name: str):
+        """Uploads local file to the H2O Drive as object."""
+        await self._bucket.upload_file(file_name, self._key_with_prefix(object_name))
+
+    async def download_file(self, object_name: str, file_name: os.PathLike):
+        """Download H2O Drive object to local file."""
+        await self._bucket.download_file(self._key_with_prefix(object_name), file_name)
+
+    async def list_objects(self, prefix: Optional[str] = None) -> List[ObjectSummary]:
+        """Returns list of available objects.
+
+        Only the objects with the prefix associated with the instance are returned and
+        the prefix is stripped. Objects can be additionally filtered by further prefix.
+        """
+        filter_prefix = self._prefix
+        if prefix is not None:
+            filter_prefix = self._key_with_prefix(prefix)
+
+        objects = await self._bucket.list_objects(prefix=filter_prefix)
+
+        result: List[ObjectSummary] = []
+        for o in objects:
+            obj_name = o.key
+            if obj_name.startswith(self._prefix):
+                obj_name = obj_name[self._prefix_length :]
+            result.append(
+                ObjectSummary(key=obj_name, size=o.size, last_modified=o.last_modified)
+            )
+
+        return result
+
+    async def delete_object(self, object_name: str):
+        """Deletes H2O Drive object."""
+        await self._bucket.delete_object(self._key_with_prefix(object_name))
+
+    async def generate_presigned_url(
+        self, object_name: str, *, ttl_seconds: int = _DEFAULT_PRESIGNED_URL_TTL_SECONDS
+    ) -> PresignedURL:
+        """Generates pre-signed URL for the H2O Drive object.
+
+        Generated URL might not be accessible externally.
+        """
+        return await self._bucket.generate_presigned_url(
+            object_name=self._key_with_prefix(object_name), ttl_seconds=ttl_seconds
+        )
+
+    def _key_with_prefix(self, key: str) -> str:
+        return str(pathlib.Path(self._prefix) / key)
+
+
+class Bucket:
+    def __init__(
+        self, name: str, *, session: auth.SessionProvider, endpoint_url: str
+    ) -> None:
+        self._name = name
+        self._session = session
+        self._endpoint_url = endpoint_url
+
+    async def upload_file(self, file_name: os.PathLike, object_name: str):
+        """Uploads local file to the H2O Drive as object in the bucket."""
+        async with self._s3_client() as s3:
+            await s3.upload_file(
+                file_name,
+                self._name,
+                object_name,
+                Config=boto3.s3.transfer.TransferConfig(max_io_queue=5),
+            )
+
+    async def download_file(self, object_name: str, file_name: os.PathLike):
+        """Download H2O Drive object in the bucket to local file."""
+        async with self._s3_client() as s3:
+            await s3.download_file(self._name, object_name, file_name)
+
+    async def list_objects(self, prefix: Optional[str] = None) -> List[ObjectSummary]:
+        """Returns list of objects available in the bucket.
+
+        Files can be filtered by the prefix.
+        """
+        async with self._s3_resource() as s3:
+            bucket = await s3.Bucket(self._name)
+            if prefix is not None:
+                objects = bucket.objects.filter(Prefix=prefix)
+            else:
+                objects = bucket.objects.all()
+            result: List[ObjectSummary] = []
+            async for o in objects:
+                result.append(
+                    ObjectSummary(
+                        key=o.key,
+                        size=await o.size,
+                        last_modified=await o.last_modified,
+                    )
+                )
+            return result
+
+    async def delete_object(self, object_name: str):
+        """Deletes H2O Drive object in the bucket."""
+        async with self._s3_resource() as s3:
+            bucket = await s3.Bucket(self._name)
+            obj = await bucket.Object(object_name)
+            await obj.delete()
+
+    async def generate_presigned_url(
+        self, object_name: str, *, ttl_seconds: int = _DEFAULT_PRESIGNED_URL_TTL_SECONDS
+    ) -> PresignedURL:
+        """Generates pre-signed URL for the H2O Drive object in the bucket.
+
+        Generated URL might not be accessible externally.
+        """
+
+        async with self._s3_client() as s3:
+            url_str = await s3.generate_presigned_url(
+                ClientMethod="get_object",
+                Params={"Bucket": self._name, "Key": object_name},
+                ExpiresIn=ttl_seconds,
+            )
+            return PresignedURL(url_str)
+
+    async def create(self) -> "Bucket":
+        """Creates the bucket."""
+        async with self._s3_resource() as s3:
+            bucket = await s3.Bucket(self._name)
+            await bucket.create()
+        return self
+
+    async def ensure_created(self) -> "Bucket":
+        """Creates the bucket if it does not exist."""
+        try:
+            await self.create()
+        except botocore.exceptions.ClientError as error:
+            if error.response["Error"]["Code"] != "BucketAlreadyExists":
+                raise
+
+        return self
+
+    def with_prefix(self, prefix: str) -> PrefixClient:
+        """Returns a new instance of the Drive client operating within the bucket but
+        with all object operations prefixed with the prefix.
+        """
+        return PrefixClient(bucket=self, prefix=prefix)
+
+    def workspace(self, workspace: Workspace) -> PrefixClient:
+        """Returns new instance og Drive client operating withing the bucket but
+        with all object operations prefixed with prefix pre-defined by workspace.
+        """
+        if workspace == Workspace.HOME:
+            prefix = _HOME_PREFIX
+        elif workspace == Workspace.APP:
+            prefix = _get_app_workspace_prefix()
+        elif workspace == Workspace.APP_VERSION:
+            prefix = _get_app_version_workspace_prefix()
+        elif workspace == Workspace.APP_INSTANCE:
+            prefix = _get_app_instance_workspace_prefix()
+        else:
+            raise ValueError(f"Unsupported workspace: {workspace}.")
+
+        return self.with_prefix(prefix=prefix)
+
+    def home(self) -> PrefixClient:
+        """Returns new instance og Drive client operating withing the bucket but
+        with all object operations prefixed with prefix pre-defined by HOME workspace.
+        """
+        return self.workspace(workspace=Workspace.HOME)
+
+    @contextlib.asynccontextmanager
+    async def _s3_client(self):
+        session = await self._session()
+        async with session.client(
+            "s3",
+            endpoint_url=self._endpoint_url,
+            config=botocore.client.Config(signature_version="s3v4"),
+        ) as s3:
+            yield s3
+
+    @contextlib.asynccontextmanager
+    async def _s3_resource(self):
+        session = await self._session()
+        async with session.resource(
+            "s3",
+            endpoint_url=self._endpoint_url,
+            config=botocore.client.Config(signature_version="s3v4"),
+        ) as s3:
+            yield s3
+
+
+class Client:
+    def __init__(self, session: auth.SessionProvider, endpoint_url: str) -> None:
+        self._session = session
+        self._endpoint_url = endpoint_url
+
+    def bucket(self, name) -> Bucket:
+        """Returns a new instance of the Bucket client."""
+        return Bucket(name=name, session=self._session, endpoint_url=self._endpoint_url)
+
+    def my_bucket(self) -> Bucket:
+        """Returns a new instance of the Bucket client for the user's personal
+        bucket."""
+        return Bucket(
+            name=MY_BUCKET, session=self._session, endpoint_url=self._endpoint_url
+        )
+
+
+def _get_app_workspace_prefix():
+    app = _get_app_name()
+    return f"{app}/workspace/"
+
+
+def _get_app_version_workspace_prefix():
+    app_version_prefix = _get_app_version_prefix()
+    return f"{app_version_prefix}/workspace/"
+
+
+def _get_app_version_prefix():
+    app, version, app_id = _get_app_name(), _get_app_version(), _get_app_id()
+    return f"{app}/{app_id}-{version}"
+
+
+def _get_app_instance_workspace_prefix():
+    app_version_prefix = _get_app_version_prefix()
+    app_instance_id = _get_app_instance_id()
+    return f"{app_version_prefix}/{app_instance_id}/workspace/"
+
+
+def _get_app_name() -> str:
+    app = os.getenv("H2O_CLOUD_APP_NAME")
+    if not app:
+        raise LookupError(
+            "Unable to determine app name. "
+            "H2O_CLOUD_APP_NAME environment variable not set."
+        )
+    return app
+
+
+def _get_app_version() -> str:
+    version = os.getenv("H2O_CLOUD_APP_VERSION")
+    if not version:
+        raise LookupError(
+            "Unable to determine app version. "
+            "H2O_CLOUD_APP_VERSION environment variable not set."
+        )
+    return version
+
+
+def _get_app_id() -> str:
+    app_id = os.getenv("H2O_CLOUD_APP_ID")
+    if not app_id:
+        raise LookupError(
+            "Unable to determine app ID. "
+            "H2O_CLOUD_APP_ID environment variable not set."
+        )
+    return app_id
+
+
+def _get_app_instance_id() -> str:
+    instance_id = os.getenv("H2O_CLOUD_INSTANCE_ID")
+    if not instance_id:
+        raise LookupError(
+            "Unable to determine app instance ID. "
+            "H2O_CLOUD_INSTANCE_ID environment variable not set."
+        )
+    return instance_id

h2o_drive-0.0.1/pyproject.toml

@@ -0,0 +1,50 @@
+[tool.poetry]
+name = "h2o-drive"
+version = "0.0.1"
+description = "H2O Drive Python Client"
+readme = "PUBLIC_README.md"
+authors = ["H2O.ai <support@h2o.ai>"]
+classifiers = [
+  "Intended Audience :: Developers",
+  "Operating System :: OS Independent",
+  "Topic :: Internet",
+  "Topic :: Scientific/Engineering :: Artificial Intelligence",
+]
+
+[tool.poetry.dependencies]
+python = "^3.7"
+aioboto3 = "^9.3.1"
+aiobotocore = "^2.1.0"
+botocore = "1.23.24"
+typing-extensions = ">=3.7.4"
+h2o-authn = {version="^1.1.0", extras=["discovery"]}
+h2o-cloud-discovery = "^1.1.0"
+
+[tool.poetry.group.dev.dependencies]
+black = "^23.3.0"
+types-aiobotocore = {extras = ["suppport"], version = "^2.6.0"}
+flake8 = "^5.0.0"
+flake8-bugbear = "^23.3.12"
+flake8-builtins = "^2.1.0"
+flake8-deprecated = "^2.0"
+flake8-eradicate = "^1.2.0"
+flake8-mutable = "^1.2.0"
+flake8-pyproject = "^1.2.3"
+mypy = "^1.4"
+boto3-stubs = "^1.28.64"
+
+[tool.flake8]
+max-line-length = 88
+import-order-style = "appnexus"
+application-import-names = ["h2o_drive"]
+ignore = ['E203'] # Incompatible with black
+
+[tool.mypy]
+
+[[tool.mypy.overrides]]
+module = "aioboto3"
+ignore_missing_imports = true
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"