truefoundry 0.4.4rc3__tar.gz → 0.4.4rc5__tar.gz

{truefoundry-0.4.4rc3 → truefoundry-0.4.4rc5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: truefoundry
-Version: 0.4.4rc3
+Version: 0.4.4rc5
 Summary: Truefoundry CLI
 Author: Abhishek Choudhary
 Author-email: abhishek@truefoundry.com
@@ -22,6 +22,7 @@ Requires-Dist: docker (>=6.1.2,<8.0.0)
 Requires-Dist: fastapi (>=0.56.0,<0.200.0)
 Requires-Dist: filelock (>=3.8.0,<4.0.0)
 Requires-Dist: flytekit (==1.12.2) ; extra == "workflow"
+Requires-Dist: fsspec (>=2024.9.0,<2025.0.0)
 Requires-Dist: gitignorefile (>=1.1.2,<2.0.0)
 Requires-Dist: importlib-metadata (>=4.11.3,<9.0.0)
 Requires-Dist: importlib-resources (>=5.2.0,<7.0.0)

{truefoundry-0.4.4rc3 → truefoundry-0.4.4rc5}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "truefoundry"
-version = "0.4.4rc3"
+version = "0.4.4rc5"
 description = "Truefoundry CLI"
 authors = ["Abhishek Choudhary <abhishek@truefoundry.com>"]
 readme = "README.md"
@@ -54,6 +54,7 @@ pandas = [
     { version = ">=1.0.0,<3.0.0", python = "<3.10" },
     { version = ">=1.4.0,<3.0.0", python = ">=3.10" },
 ]
+fsspec = "^2024.9.0"
 
 
 [tool.poetry.extras]

{truefoundry-0.4.4rc3 → truefoundry-0.4.4rc5}/truefoundry/common/constants.py

@@ -11,12 +11,33 @@ CREDENTIAL_FILEPATH = TFY_CONFIG_DIR / "credentials.json"
 TFY_HOST_ENV_KEY = "TFY_HOST"
 TFY_API_KEY_ENV_KEY = "TFY_API_KEY"
 
+TFY_INTERNAL_SIGNED_URL_SERVER_HOST_ENV_KEY = "TFY_INTERNAL_SIGNED_URL_SERVER_HOST"
+TFY_INTERNAL_SIGNED_URL_SERVER_TOKEN_ENV_KEY = "TFY_INTERNAL_SIGNED_URL_SERVER_TOKEN"
+TFY_INTERNAL_SIGNED_URL_SERVER_DEFAULT_TTL_ENV_KEY = (
+    "TFY_INTERNAL_SIGNED_URL_SERVER_DEFAULT_TTL"
+)
+TFY_INTERNAL_SIGNED_URL_SERVER_MAX_TIMEOUT_ENV_KEY = (
+    "TFY_INTERNAL_SIGNED_URL_SERVER_MAX_TIMEOUT"
+)
+
 
 class TrueFoundrySdkEnv(BaseSettings):
     # Note: Every field in this class should have a default value
     # Never expect the user to set these values
     TFY_HOST: Optional[str] = Field(default=None, env=TFY_HOST_ENV_KEY)
     TFY_API_KEY: Optional[SecretStr] = Field(default=None, env=TFY_API_KEY_ENV_KEY)
+    TFY_INTERNAL_SIGNED_URL_SERVER_HOST: Optional[str] = Field(
+        default=None, env=TFY_INTERNAL_SIGNED_URL_SERVER_HOST_ENV_KEY
+    )
+    TFY_INTERNAL_SIGNED_URL_SERVER_TOKEN: Optional[str] = Field(
+        default=None, env=TFY_INTERNAL_SIGNED_URL_SERVER_TOKEN_ENV_KEY
+    )
+    TFY_INTERNAL_SIGNED_URL_SERVER_DEFAULT_TTL: int = Field(
+        default=3600, env=TFY_INTERNAL_SIGNED_URL_SERVER_DEFAULT_TTL_ENV_KEY
+    )  # 1 hour
+    TFY_INTERNAL_SIGNED_URL_SERVER_MAX_TIMEOUT: int = Field(
+        default=5, env=TFY_INTERNAL_SIGNED_URL_SERVER_MAX_TIMEOUT_ENV_KEY
+    )  # 5 seconds
     TFY_ARTIFACTS_DOWNLOAD_CHUNK_SIZE_BYTES: int = 100 * 1000 * 1000
     TFY_ARTIFACTS_DOWNLOAD_MAX_WORKERS: int = max(min(32, (os.cpu_count() or 2) * 2), 4)
     TFY_ARTIFACTS_UPLOAD_MAX_WORKERS: int = max(min(32, (os.cpu_count() or 2) * 2), 4)

truefoundry-0.4.4rc5/truefoundry/common/tfy_signed_url_client.py

@@ -0,0 +1,251 @@
+# file: client.py
+from enum import Enum
+from typing import Any, Dict, List, Optional
+from urllib.parse import urlencode, urljoin
+
+import requests
+from requests.exceptions import RequestException
+
+from truefoundry.common.constants import ENV_VARS
+from truefoundry.pydantic_v1 import BaseModel, Field
+
+DEFAULT_TTL = ENV_VARS.TFY_INTERNAL_SIGNED_URL_SERVER_DEFAULT_TTL
+MAX_TIMEOUT = ENV_VARS.TFY_INTERNAL_SIGNED_URL_SERVER_MAX_TIMEOUT
+
+
+class SignedURLAPIResponseDto(BaseModel):
+    uri: str
+    signed_url: str
+    headers: Optional[Dict[str, Any]] = None
+
+
+class SignedURLIsDirectoryAPIResponseDto(BaseModel):
+    is_directory: bool = Field(..., alias="isDirectory")
+
+
+class SignedURLExistsAPIResponseDto(BaseModel):
+    exists: bool
+
+
+class FileInfo(BaseModel):
+    path: str
+    is_directoy: bool = Field(..., alias="isDirectory")
+    bytes: Optional[int] = None
+    signedUrl: Optional[str] = None
+
+
+class PagedList(BaseModel):
+    items: List[FileInfo]
+    token: Optional[str] = None
+
+
+class SignedURLServerEndpoint(str, Enum):
+    """Enumeration for Signed URL Server endpoints."""
+
+    READ = "/v1/signed-uri/read"
+    WRITE = "/v1/signed-uri/write"
+    EXISTS = "/v1/exists"
+    IS_DIRECTORY = "/v1/is-dir"
+    LIST_FILES = "/v1/list-files"
+
+
+class SignedURLClient:
+    """Client to interact with the Signed URL Server for file operations."""
+
+    def __init__(
+        self,
+        base_url: Optional[str] = None,
+        token: Optional[str] = None,
+        ttl: int = DEFAULT_TTL,
+    ):
+        """
+        Initialize the SignedURLClient.
+
+        Args:
+            base_url (str): Base URL of the signed URL server (optional).
+            token (str): Token for authentication (optional).
+            ttl (int): Default time-to-live for signed URLs in seconds.
+        """
+        self.base_url = base_url or ENV_VARS.TFY_INTERNAL_SIGNED_URL_SERVER_HOST
+        self.token = token or ENV_VARS.TFY_INTERNAL_SIGNED_URL_SERVER_TOKEN
+        self.ttl = ttl
+
+        self.headers = {
+            "Authorization": f"Bearer {self.token}",
+            "Content-Type": "application/json",
+        }
+
+    def _make_request(
+        self, endpoint: str, method: str = "GET", payload: Optional[Dict] = None
+    ) -> Dict:
+        """
+        Internal method to handle requests to the signed URL server.
+
+        Args:
+            endpoint (str): The endpoint for the request.
+            method (str): HTTP method (GET, POST, etc.).
+            payload (Dict, optional): JSON payload for the request.
+
+        Returns:
+            Dict: JSON response from the server.
+
+        Raises:
+            RuntimeError: For network errors or invalid responses.
+        """
+        url = urljoin(self.base_url, endpoint)
+        try:
+            response = requests.request(
+                method, url, headers=self.headers, json=payload, timeout=MAX_TIMEOUT
+            )
+            response.raise_for_status()
+            return response.json()
+        except RequestException as e:
+            raise RuntimeError(f"Error during request to {url}: {e}") from e
+
+    def _make_server_api_call(
+        self, endpoint: SignedURLServerEndpoint, params: Optional[Dict] = None
+    ) -> Dict:
+        """Get a signed URL for the specified operation and URI."""
+        query_string = urlencode(params or {})
+        endpoint_with_params = f"{endpoint.value}?{query_string}"
+        return self._make_request(endpoint_with_params)
+
+    def _upload_data(self, signed_url: str, data: Any) -> None:
+        """Helper method to upload data to the signed URL."""
+        try:
+            # TODO: Add TIMEOUT to this request
+            response = requests.put(
+                signed_url,
+                headers={"Content-Type": "application/octet-stream"},
+                data=data,
+            )
+            response.raise_for_status()
+        except Exception as e:
+            raise RuntimeError(f"Failed to upload data: {e}") from e
+
+    def upload_from_bytes(self, data: bytes, storage_uri: str) -> str:
+        """Upload bytes to the specified storage path using a signed URL."""
+        signed_object = self._make_server_api_call(
+            endpoint=SignedURLServerEndpoint.WRITE,
+            params={"uri": storage_uri, "expiryInSeconds": self.ttl},
+        )
+        pre_signed_object_dto = SignedURLAPIResponseDto.parse_obj(signed_object)
+        self._upload_data(pre_signed_object_dto.signed_url, data)
+        return storage_uri
+
+    def upload(self, file_path: str, storage_uri: str) -> str:
+        """Upload a file to the specified storage path using a signed URL."""
+        print(f"Uploading {file_path} to {storage_uri}")
+        response = self._make_server_api_call(
+            endpoint=SignedURLServerEndpoint.WRITE,
+            params={"uri": storage_uri, "expiryInSeconds": self.ttl},
+        )
+        pre_signed_object_dto = SignedURLAPIResponseDto.parse_obj(response)
+        with open(file_path, "rb") as file:
+            self._upload_data(pre_signed_object_dto.signed_url, file)
+        return storage_uri
+
+    def _download_file(
+        self, signed_url: str, local_path: Optional[str] = None
+    ) -> Optional[bytes]:
+        """Common method to download a file using a signed URL.
+
+        Args:
+            signed_url (str): The signed URL to download from.
+            local_path (Optional[str]): The local path to save the file. If None, the file will not be saved.
+
+        Returns:
+            Optional[bytes]: The content of the downloaded file if local_path is None; otherwise, None.
+        """
+        try:
+            # TODO: Add TIMEOUT to this request
+            response = requests.get(
+                signed_url,
+                stream=True,
+                headers=self.headers,
+            )
+            response.raise_for_status()
+
+            if local_path:
+                with open(local_path, "wb") as file:
+                    for chunk in response.iter_content(chunk_size=8192):
+                        file.write(chunk)
+                return None  # File saved successfully; no need to return content.
+
+            return response.content  # Return the content if not saving to a local path.
+
+        except requests.RequestException as e:
+            raise RuntimeError(f"Failed to download file from {signed_url}: {e}") from e
+
+    def download(self, storage_uri: str, local_path: str) -> Optional[str]:
+        """Download a file from the specified storage path to a local path using a signed URL.
+
+        Args:
+            storage_uri (str): The storage URI to download the file from.
+            local_path (str): The local path to save the downloaded file.
+
+        Returns:
+            Optional[str]: The local path if successful; None if the file is saved.
+        """
+        print(f"Dowloading {storage_uri} to {local_path}")
+        response = self._make_server_api_call(
+            endpoint=SignedURLServerEndpoint.READ,
+            params={"uri": storage_uri, "expiryInSeconds": self.ttl},
+        )
+        presigned_object = SignedURLAPIResponseDto.parse_obj(response)
+        self._download_file(presigned_object.signed_url, local_path)
+        return local_path  # Indicate successful download.
+
+    def download_to_bytes(self, storage_uri: str) -> bytes:
+        """Download a file from the specified storage path and return it as bytes.
+
+        Args:
+            storage_uri (str): The storage URI to download the file from.
+
+        Returns:
+            bytes: The content of the downloaded file.
+        """
+        response = self._make_server_api_call(
+            endpoint=SignedURLServerEndpoint.READ,
+            params={"uri": storage_uri, "expiryInSeconds": self.ttl},
+        )
+        presigned_object = SignedURLAPIResponseDto.parse_obj(response)
+        return self._download_file(presigned_object.signed_url)
+
+    def exists(self, uri: str) -> bool:
+        """Check if a file exists at the specified path."""
+        response = self._make_server_api_call(
+            endpoint=SignedURLServerEndpoint.EXISTS, params={"uri": uri}
+        )
+        response_obj = SignedURLExistsAPIResponseDto.parse_obj(response)
+        return response_obj.exists
+
+    def is_directory(self, uri: str) -> bool:
+        """Check if the specified URI is a directory."""
+        response = self._make_server_api_call(
+            endpoint=SignedURLServerEndpoint.IS_DIRECTORY, params={"path": uri}
+        )
+        response_obj = SignedURLIsDirectoryAPIResponseDto.parse_obj(response)
+        print(f"Path {uri} is_directory: {response_obj.is_directory}")
+        return response_obj.is_directory
+
+    def list_files(self, path: str, detail: bool = False, max_results: int = 1000):
+        """List files in the specified directory."""
+        token = ""
+        items: List[FileInfo] = []
+        # Fetch all files in the specified path, in pages of max_results
+        while True:
+            response = self._make_server_api_call(
+                endpoint=SignedURLServerEndpoint.LIST_FILES,
+                params={"path": path, "maxResults": max_results, "pageToken": token},
+            )
+            response_obj = PagedList.parse_obj(response)
+            items.extend(response_obj.items)
+            token = response_obj.token
+            if not token:
+                break
+
+        # Return the items or paths based on the detail flag
+        if detail:
+            return items
+        return [item.path for item in items]

truefoundry-0.4.4rc5/truefoundry/common/tfy_signed_url_fs.py

@@ -0,0 +1,232 @@
+# file: tfy_signed_url_fs.py
+# pylint: disable=W0223
+import io
+import os
+from pathlib import Path
+from typing import Optional
+
+from fsspec.spec import DEFAULT_CALLBACK, AbstractBufferedFile, AbstractFileSystem
+
+from truefoundry.common.constants import ENV_VARS
+from truefoundry.common.tfy_signed_url_client import SignedURLClient
+
+
+class SignedURLFileSystem(AbstractFileSystem):
+    def __init__(self, base_url: Optional[str] = None, token: Optional[str] = None):
+        super().__init__()
+        base_url = base_url or ENV_VARS.TFY_INTERNAL_SIGNED_URL_SERVER_HOST
+        token = token or ENV_VARS.TFY_INTERNAL_SIGNED_URL_SERVER_TOKEN
+        self.client = SignedURLClient(base_url, token)
+
+    def exists(self, path, **kwargs):
+        """Is there a file at the given path"""
+        return self.client.exists(path)
+
+    def get(
+        self,
+        rpath,
+        lpath,
+        recursive=False,
+        callback=DEFAULT_CALLBACK,
+        maxdepth=None,
+        **kwargs,
+    ):
+        """Get file(s) to local"""
+        # TODO: Add support for ThreadPoolExecutor here
+        # TODO: Do a proper error handling here
+        # TODO: Add support for async download
+        if self.isdir(rpath):
+            if not recursive:
+                raise ValueError(
+                    f"{rpath} is a directory, but recursive is not enabled."
+                )
+
+            # Handle recursive download
+            files = self.ls(rpath, detail=True)
+
+            for file_info in files:
+                file_path = file_info.path.rstrip("/").rsplit("/")[-1]
+
+                is_directory = file_info.is_directoy
+                # Construct the relative path for local download
+                relative_path = rpath.rstrip("/") + "/" + file_path
+                target_local_path = lpath.rstrip("/") + "/" + file_path
+
+                if is_directory:
+                    # If it's a directory, create the directory locally
+                    Path(target_local_path).mkdir(parents=True, exist_ok=True)
+                    relative_path = relative_path + "/"
+                    if recursive:
+                        # Recursively download the contents of the directory
+                        self.get(
+                            relative_path,
+                            target_local_path,
+                            recursive=True,
+                            maxdepth=maxdepth,
+                            **kwargs,
+                        )
+                else:
+                    # Download the file
+                    self.client.download(
+                        storage_uri=relative_path, local_path=target_local_path
+                    )
+
+        else:
+            target_local_path = lpath
+            if target_local_path.endswith("/"):
+                Path(target_local_path).mkdir(parents=True, exist_ok=True)
+                target_local_path = target_local_path + rpath.rsplit("/")[-1]
+            self.client.download(storage_uri=rpath, local_path=target_local_path)
+
+    def put(
+        self,
+        lpath,
+        rpath,
+        recursive=False,
+        callback=DEFAULT_CALLBACK,
+        maxdepth=None,
+        **kwargs,
+    ):
+        local_path = Path(lpath)
+        if local_path.is_dir():
+            if not recursive:
+                raise ValueError(
+                    f"{lpath} is a directory, but recursive is set to False."
+                )
+
+            # Optionally limit recursion depth
+            max_depth = maxdepth if maxdepth is not None else float("inf")
+
+            # Walk through the directory structure
+            for root, _, files in os.walk(lpath):
+                current_depth = Path(root).relative_to(local_path).parts
+                if len(current_depth) > max_depth:
+                    continue  # Skip files deeper than the max depth
+
+                rel_dir = Path(root).relative_to(local_path)
+                remote_dir = (
+                    rpath.rstrip("/")
+                    if rel_dir == Path(".")
+                    else rpath.rstrip("/") + "/" + str(rel_dir)
+                )
+
+                # Upload each file
+                for file in files:
+                    local_file_path = Path(root) / file
+                    remote_file_path = f"{remote_dir}/{file}"
+                    self.client.upload(
+                        file_path=str(local_file_path),
+                        storage_uri=str(remote_file_path),
+                    )
+            return None
+        else:
+            # Handle single file upload
+            return self.client.upload(
+                file_path=str(local_path), storage_uri=str(Path(rpath))
+            )
+
+    def isdir(self, path):
+        """Is this entry directory-like?"""
+        return self.client.is_directory(path)
+
+    def open(
+        self,
+        path,
+        mode="rb",
+        block_size=None,
+        cache_options=None,
+        compression=None,
+        **kwargs,
+    ):
+        """
+        Open a file for reading or writing.
+        """
+        if "r" in mode:
+            # Reading mode
+            file_content = self.client.download_to_bytes(path)
+            return io.BytesIO(file_content)
+        elif "w" in mode or "a" in mode:
+            # Writing mode (appending treated as writing)
+            buffer = io.BytesIO()
+            buffer.seek(0)
+
+            def on_close(buffer=buffer, path=path):
+                """
+                Callback when file is closed, automatically upload the content.
+                """
+                buffer.seek(0)
+                self.client.upload_from_bytes(buffer.read(), storage_uri=path)
+
+            # Wrapping the buffer to automatically upload on close
+            return io.BufferedWriter(buffer, on_close)
+
+    def write(self, path, data, **kwargs):
+        """
+        Write data to the file at the specified path (this could be tied to open's close).
+        """
+        if isinstance(data, io.BytesIO):
+            data.seek(0)
+            content = data.read()
+        elif isinstance(data, str):
+            content = data.encode()  # Encode to bytes
+        else:
+            raise ValueError("Unsupported data type for writing")
+
+        # Upload the content to the remote file system
+        self.client.upload_from_bytes(content, storage_uri=path)
+
+    def ls(self, path, detail=True, **kwargs):
+        """List objects at path."""
+        return self.client.list_files(path, detail=detail)
+
+
+class SignedURLBufferedFile(AbstractBufferedFile):
+    """
+    Buffered file implementation for Signed URL-based file system.
+    """
+
+    def __init__(
+        self, fs: SignedURLFileSystem, path: str, mode: str, block_size: int, **kwargs
+    ):
+        """
+        Initialize the buffered file, determining the mode (read/write).
+        """
+        super().__init__(fs, path, mode, block_size, **kwargs)
+        self.buffer = io.BytesIO()
+        self.client = fs.client
+
+        if "r" in mode:
+            # Download the file content for reading
+            file_content = fs.client.download_to_bytes(path)
+            self.buffer.write(file_content)
+            self.buffer.seek(0)  # Reset buffer after writing content
+
+    def _upload_on_close(self):
+        """
+        Upload content back to the remote store when the file is closed.
+        """
+        self.buffer.seek(0)
+        self.client.upload_from_bytes(self.buffer.read(), storage_uri=self.path)
+
+    def close(self):
+        """
+        Close the file, ensuring the content is uploaded for write/append modes.
+        """
+        if self.writable():
+            self._upload_on_close()
+        self.buffer.close()
+        super().close()
+
+    def _fetch_range(self, start, end):
+        """
+        Fetch a specific byte range from the file. Useful for large files and range reads.
+        """
+        self.buffer.seek(start)
+        return self.buffer.read(end - start)
+
+    def _upload_chunk(self, final=False):
+        """
+        Upload a chunk of the file. For larger files, data may be uploaded in chunks.
+        """
+        if final:
+            self._upload_on_close()

{truefoundry-0.4.4rc3 → truefoundry-0.4.4rc5}/truefoundry/deploy/python_deploy_codegen.py

@@ -107,16 +107,16 @@ def convert_deployment_config_to_python(workspace_fqn: str, application_spec: di
     Convert a deployment config to a python file that can be used to deploy to a workspace
     """
     application = Application.parse_obj(application_spec)
-    application_type = application.__root__.type
-
+    application_obj = application.__root__
+    application_type = application_obj.type
     if (
-        hasattr(application.__root__, "image")
-        and application.__root__.image.type == "build"
-        and application.__root__.image.build_source.type == "remote"
+        hasattr(application_obj, "image")
+        and application_obj.image.type == "build"
+        and application_obj.image.build_source.type == "remote"
     ):
-        application.__root__.image.build_source = LocalSource(local_build=False)
+        application_obj.image.build_source = LocalSource(local_build=False)
 
-    spec_repr = get_python_repr(application.__root__)
+    spec_repr = get_python_repr(application_obj)
     spec_repr = replace_enums_with_values(spec_repr)
     spec_repr = remove_none_type_fields(spec_repr)
     spec_repr = remove_type_field(spec_repr)
@@ -144,7 +144,8 @@ def convert_deployment_config_to_python(workspace_fqn: str, application_spec: di
 def generate_python_snippet_for_trigger_job(
     application_fqn: str, command: Optional[str], params: Optional[Dict[str, str]]
 ):
-    job_run_python_template = """from truefoundry.deploy import trigger_job
+    job_run_python_template = """\
+from truefoundry.deploy import trigger_job
 
 response = trigger_job(
   application_fqn="{{application_fqn}}",
@@ -160,16 +161,16 @@ print(response.jobRunName)
     )
 
     if command is not None:
-        output_python_str = output_python_str.replace("{{command}}", repr(command))
         output_python_str = output_python_str.replace("# command", "command")
+        output_python_str = output_python_str.replace("{{command}}", repr(command))
     else:
         output_python_str = output_python_str.replace(
             "{{command}}", "<Enter Command Here>"
         )
 
     if params is not None:
-        output_python_str = output_python_str.replace("{{params}}", repr(params))
         output_python_str = output_python_str.replace("# params", "params")
+        output_python_str = output_python_str.replace("{{params}}", repr(params))
     else:
         output_python_str = output_python_str.replace(
             "{{params}}", "<Enter Params(key-value pairs) here as python dict>"
@@ -195,7 +196,7 @@ def generate_curl_snippet_for_trigger_job(
 }'
 """
     output_curl_str = job_run_curl_request_template.replace(
-        "{{control_plane_url}}", control_plane_url
+        "{{control_plane_url}}", control_plane_url.rstrip("/")
     )
     output_curl_str = output_curl_str.replace("{{application_id}}", application_id)
     output_curl_str = output_curl_str.replace(

{truefoundry-0.4.4rc3 → truefoundry-0.4.4rc5}/truefoundry/workflow/__init__.py

@@ -1,4 +1,5 @@
 try:
+    import fsspec
     from flytekit import task as _
 except ImportError:
     print("To use workflows, please run 'pip install truefoundry[workflow]'.")
@@ -6,6 +7,7 @@ except ImportError:
 from flytekit import conditional
 from flytekit.types.directory import FlyteDirectory
 
+from truefoundry.common.tfy_signed_url_fs import SignedURLFileSystem
 from truefoundry.deploy.v2.lib.patched_models import (
     ContainerTaskConfig,
     PythonTaskConfig,
@@ -32,3 +34,6 @@ __all__ = [
     "PythonTaskConfig",
     "ExecutionConfig",
 ]
+print("SignedURL FS Spec registered for s3 and gs")
+fsspec.register_implementation("s3", SignedURLFileSystem, clobber=True)
+fsspec.register_implementation("gs", SignedURLFileSystem, clobber=True)