wmill 1.258.4__tar.gz → 1.259.1__tar.gz

{wmill-1.258.4 → wmill-1.259.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: wmill
-Version: 1.258.4
+Version: 1.259.1
 Summary: A client library for accessing Windmill server wrapping the Windmill client API
 Home-page: https://windmill.dev
 License: Apache-2.0

{wmill-1.258.4 → wmill-1.259.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "wmill"
-version = "1.258.4"
+version = "1.259.1"
 description = "A client library for accessing Windmill server wrapping the Windmill client API"
 license = "Apache-2.0"
 homepage = "https://windmill.dev"

{wmill-1.258.4 → wmill-1.259.1}/wmill/client.py

@@ -386,7 +386,8 @@ class Windmill:
         file_content = my_obj_content.decode("utf-8")
         '''
         """
-        return self.load_s3_file_reader(s3object, s3_resource_path).read()
+        with self.load_s3_file_reader(s3object, s3_resource_path) as file_reader:
+            return file_reader.read()
 
     def load_s3_file_reader(self, s3object: S3Object, s3_resource_path: str | None) -> BufferedReader:
         """
@@ -396,24 +397,17 @@ class Windmill:
         from wmill import S3Object
 
         s3_obj = S3Object(s3="/path/to/my_file.txt")
-        my_obj_content_reader = client.load_s3_file_reader(s3_obj)
-        file_content = my_obj_content_reader.read().decode("utf-8")
+        with wmill.load_s3_file(s3object, s3_resource_path) as file_reader:
+            print(file_reader.read())
         '''
         """
-
-        result = S3BufferedReader(
-            workspace=f"{self.workspace}",
-            windmill_client=self.client,
-            file_key=s3object["s3"],
-            s3_resource_path=s3_resource_path,
-        )
-        return result
+        reader = S3BufferedReader(f"{self.workspace}", self.client, s3object["s3"], s3_resource_path)
+        return reader
 
     def write_s3_file(
         self,
         s3object: S3Object | None,
         file_content: BufferedReader | bytes,
-        file_expiration: dt.datetime | None,
         s3_resource_path: str | None,
     ) -> S3Object:
         """
@@ -441,38 +435,25 @@ class Windmill:
         else:
             raise Exception("Type of file_content not supported")
 
-        file_key = s3object["s3"] if s3object is not None else None
-        parts = []
-        upload_id = None
-        chunk = content_reader.read(5 * 1024 * 1024)
-        if len(chunk) == 0:
-            raise Exception("File content is empty, nothing to upload")
-        while True:
-            chunk_2 = content_reader.read(5 * 1024 * 1024)
-            reader_done = len(chunk_2) == 0
-            try:
-                response = self.post(
-                    f"/w/{self.workspace}/job_helpers/multipart_upload_s3_file",
-                    json={
-                        "file_key": file_key,
-                        "part_content": [b for b in chunk],
-                        "upload_id": upload_id,
-                        "parts": parts,
-                        "is_final": reader_done,
-                        "cancel_upload": False,
-                        "s3_resource_path": s3_resource_path,
-                        "file_expiration": file_expiration.isoformat() if file_expiration else None,
-                    },
-                ).json()
-            except Exception as e:
-                raise Exception("Could not write file to S3") from e
-            parts = response["parts"]
-            upload_id = response["upload_id"]
-            file_key = response["file_key"]
-            if response["is_done"]:
-                break
-            chunk = chunk_2
-        return S3Object(s3=file_key)
+        query_params = {}
+        if s3object is not None and s3object["s3"] != "":
+            query_params["file_key"] = s3object["s3"]
+        if s3_resource_path is not None and s3_resource_path != "":
+            query_params["s3_resource_path"] = s3_resource_path
+
+        try:
+            # need a vanilla client b/c content-type is not application/json here
+            response = httpx.post(
+                f"{self.base_url}/w/{self.workspace}/job_helpers/upload_s3_file",
+                headers={"Authorization": f"Bearer {self.token}", "Content-Type": "application/octet-stream"},
+                params=query_params,
+                content=content_reader,
+                verify=self.verify,
+                timeout=None,
+            ).json()
+        except Exception as e:
+            raise Exception("Could not write file to S3") from e
+        return S3Object(s3=response["file_key"])
 
     def __boto3_connection_settings(self, s3_resource) -> Boto3ConnectionSettings:
         endpoint_url_prefix = "https://" if s3_resource["useSSL"] else "http://"
@@ -727,7 +708,7 @@ def boto3_connection_settings(s3_resource_path: str = "") -> Boto3ConnectionSett
 @init_global_client
 def load_s3_file(s3object: S3Object, s3_resource_path: str = "") -> bytes:
     """
-    Load the entire content of a file stored in S3
+    Load the entire content of a file stored in S3 as bytes
     """
     return _client.load_s3_file(s3object, s3_resource_path if s3_resource_path != "" else None)
 
@@ -735,7 +716,7 @@ def load_s3_file(s3object: S3Object, s3_resource_path: str = "") -> bytes:
 @init_global_client
 def load_s3_file_reader(s3object: S3Object, s3_resource_path: str = "") -> BufferedReader:
     """
-    Load the content of a file stored in S3 as a buffered reader
+    Load the content of a file stored in S3
     """
     return _client.load_s3_file_reader(s3object, s3_resource_path if s3_resource_path != "" else None)
 
@@ -744,15 +725,12 @@ def load_s3_file_reader(s3object: S3Object, s3_resource_path: str = "") -> Buffe
 def write_s3_file(
     s3object: S3Object | None,
     file_content: BufferedReader | bytes,
-    file_expiration: dt.datetime | None = None,
     s3_resource_path: str = "",
 ) -> S3Object:
     """
     Upload a file to S3
     """
-    return _client.write_s3_file(
-        s3object, file_content, file_expiration, s3_resource_path if s3_resource_path != "" else None
-    )
+    return _client.write_s3_file(s3object, file_content, s3_resource_path if s3_resource_path != "" else None)
 
 
 @init_global_client

wmill-1.259.1/wmill/s3_reader.py

@@ -0,0 +1,48 @@
+from io import BufferedReader
+from json import JSONDecodeError
+
+import httpx
+
+
+class S3BufferedReader(BufferedReader):
+    def __init__(self, workspace: str, windmill_client: httpx.Client, file_key: str, s3_resource_path: str | None):
+        params = {
+            "file_key": file_key,
+        }
+        if s3_resource_path is not None:
+            params["s3_resource_path"] = s3_resource_path
+        self._context_manager = windmill_client.stream(
+            "GET",
+            f"/w/{workspace}/job_helpers/download_s3_file",
+            params=params,
+            timeout=None,
+        )
+
+    def __enter__(self):
+        reader = self._context_manager.__enter__()
+        self._iterator = reader.iter_bytes()
+        return self
+
+    def peek(self, size=0):
+        raise Exception("Not implemented, use read() instead")
+
+    def read(self, size=-1):
+        read_result = []
+        if size < 0:
+            for b in self._iterator:
+                read_result.append(b)
+        else:
+            for i in range(size):
+                try:
+                    b = self._iterator.__next__()
+                except StopIteration:
+                    break
+                read_result.append(b)
+
+        return b''.join(read_result)
+
+    def read1(self, size=-1):
+        return self.read(size)
+
+    def __exit__(self, *args):
+        self._context_manager.__exit__(*args)

wmill-1.258.4/wmill/s3_reader.py

@@ -1,149 +0,0 @@
-from io import BufferedReader
-from json import JSONDecodeError
-
-import httpx
-
-
-class S3BufferedReader(BufferedReader):
-    def __init__(self, workspace: str, windmill_client: httpx.Client, file_key: str, s3_resource_path: str | None):
-        self._workspace = workspace
-        self._client = windmill_client
-        self._file_key = file_key
-        self._s3_resource_path = s3_resource_path
-        self._file_size: int | None = None
-
-        self._part_number: int | None = 0
-        self._current_chunk: list[int] = []
-        self._position_in_chunk = 0
-
-    def peek(self, size=0):
-        read_result = []
-
-        if size > 0 or (
-            len(self._current_chunk) > self._position_in_chunk
-            and len(self._current_chunk) > self._position_in_chunk + size
-        ):
-            payload_to_return = self._current_chunk[self._position_in_chunk : (self._position_in_chunk + size)]
-            read_result += payload_to_return
-            return bytes(read_result)
-
-        if self._position_in_chunk < len(self._current_chunk):
-            payload_to_return = self._current_chunk[self._position_in_chunk :]
-            read_result += bytes(payload_to_return)
-
-        previous_chunk = self._current_chunk
-        previous_part_number = self._part_number
-        previous_position_in_chunk = self._position_in_chunk
-        try:
-            while len(read_result) < size or self._part_number is not None:
-                self._download_new_chunk()
-                if size > 0 and size - len(read_result) < len(self._current_chunk):
-                    payload_to_return = self._current_chunk[: (size - len(read_result))]
-                    self._position_in_chunk = size - len(read_result)
-                    read_result += bytes(payload_to_return)
-                    break
-
-                read_result += bytes(self._current_chunk)
-                if self._part_number is None:
-                    break
-        finally:
-            # always roll back the changes to the stream state
-            self._current_chunk = previous_chunk
-            self._part_number = previous_part_number
-            self._position_in_chunk = previous_position_in_chunk
-        return read_result
-
-    def read(self, size=-1):
-        read_result = []
-
-        if size > 0 and (
-            len(self._current_chunk) > self._position_in_chunk
-            and len(self._current_chunk) > self._position_in_chunk + size
-        ):
-            payload_to_return = self._current_chunk[self._position_in_chunk : (self._position_in_chunk + size)]
-            self._position_in_chunk += size
-            read_result += payload_to_return
-            return bytes(read_result)
-
-        if self._position_in_chunk < len(self._current_chunk):
-            payload_to_return = self._current_chunk[self._position_in_chunk :]
-            self._position_in_chunk = len(self._current_chunk)
-            read_result += payload_to_return
-
-        previous_chunk = self._current_chunk
-        previous_part_number = self._part_number
-        previous_position_in_chunk = self._position_in_chunk
-        try:
-            while len(read_result) < size or self._part_number is not None:
-                self._download_new_chunk()
-                if size > 0 and size - len(read_result) < len(self._current_chunk):
-                    payload_to_return = self._current_chunk[: (size - len(read_result))]
-                    self._position_in_chunk = size - len(read_result)
-                    read_result += payload_to_return
-                    break
-
-                read_result += self._current_chunk
-                if self._part_number is None:
-                    break
-        except Exception as e:
-            # roll back the changes to the stream state
-            self._current_chunk = previous_chunk
-            self._part_number = previous_part_number
-            self._position_in_chunk = previous_position_in_chunk
-            raise e
-        return bytes(read_result)
-
-    def read1(self, size=-1):
-        read_result = []
-
-        if size < 0:
-            payload_to_return = self._current_chunk[self._position_in_chunk :]
-            self._position_in_chunk = len(self._current_chunk)
-            read_result += payload_to_return
-            return bytes(read_result)
-
-        if size > 0 and len(self._current_chunk) > self._position_in_chunk:
-            end_byte = min(self._position_in_chunk + size, len(self._current_chunk))
-            payload_to_return = self._current_chunk[self._position_in_chunk : end_byte]
-            self._position_in_chunk = end_byte
-            read_result += payload_to_return
-            return bytes(read_result)
-
-        # no bytes in current buffer, load a new chunk
-        self._download_new_chunk()
-        end_byte = min(size, len(self._current_chunk))
-        payload_to_return = self._current_chunk[:end_byte]
-        self._position_in_chunk = end_byte
-        read_result += payload_to_return
-        return bytes(read_result)
-
-    def close(self):
-        self._part_number = 0
-        self._current_chunk = []
-        self._position_in_chunk = 0
-
-    def _download_new_chunk(
-        self,
-    ):
-        try:
-            raw_response = self._client.post(
-                f"/w/{self._workspace}/job_helpers/multipart_download_s3_file",
-                json={
-                    "file_key": self._file_key,
-                    "part_number": self._part_number,
-                    "file_size": self._file_size,
-                    "s3_resource_path": self._s3_resource_path,
-                },
-            )
-            try:
-                raw_response.raise_for_status()
-            except httpx.HTTPStatusError as err:
-                raise Exception(f"{err.request.url}: {err.response.status_code}, {err.response.text}")
-            response = raw_response.json()
-        except JSONDecodeError as e:
-            raise Exception("Could not generate download S3 file part") from e
-
-        self._current_chunk = response["part_content"]
-        self._part_number = response["next_part_number"]
-        self._file_size = response["file_size"]
-        self._position_in_chunk = 0