tosnativeclient 1.0.2__tar.gz → 1.0.5__tar.gz

{tosnativeclient-1.0.2 → tosnativeclient-1.0.5}/Cargo.lock

@@ -1521,7 +1521,7 @@ dependencies = [
 
 [[package]]
 name = "tosnativeclient"
-version = "1.0.2"
+version = "1.0.5"
 dependencies = [
  "arc-swap",
  "async-channel",

{tosnativeclient-1.0.2 → tosnativeclient-1.0.5}/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "tosnativeclient"
-version = "1.0.2"
+version = "1.0.5"
 edition = "2021"
 publish = false
 authors = ["xiangshijian"]

{tosnativeclient-1.0.2 → tosnativeclient-1.0.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tosnativeclient
-Version: 1.0.2
+Version: 1.0.5
 Classifier: Programming Language :: Rust
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy

{tosnativeclient-1.0.2 → tosnativeclient-1.0.5}/python/tosnativeclient/tosnativeclient.pyi

@@ -31,6 +31,8 @@ class ListStream(object):
     prefix: str
     delimiter: str
     max_keys: int
+    continuation_token: str
+    start_after: str
 
     def __iter__(self) -> ListStream: ...
 
@@ -38,6 +40,10 @@ class ListStream(object):
 
     def close(self) -> None: ...
 
+    def current_prefix(self) -> Optional[str]: ...
+
+    def current_continuation_token(self) -> Optional[str]: ...
+
 
 class ReadStream(object):
     bucket: str
@@ -45,7 +51,7 @@ class ReadStream(object):
     size: int
     etag: str
 
-    def read(self, offset: int, length: int) -> bytes:
+    def read(self, offset: int, length: int) -> Optional[bytes]:
         ...
 
     def close(self) -> None:
@@ -72,13 +78,18 @@ class TosClient(object):
     part_size: int
     max_retry_count: int
     max_prefetch_tasks: int
+    directives: str
+    directory: str
+    file_name_prefix: str
+    shared_prefetch_tasks: int
 
     def __init__(self, region: str, endpoint: str, ak: str = '', sk: str = '', part_size: int = 8388608,
                  max_retry_count: int = 3, max_prefetch_tasks: int = 3, directives: str = '', directory: str = '',
                  file_name_prefix: str = '', shared_prefetch_tasks: int = 20):
         ...
 
-    def list_objects(self, bucket: str, prefix: str = '', max_keys: int = 1000, delimiter: str = '') -> ListStream:
+    def list_objects(self, bucket: str, prefix: str = '', max_keys: int = 1000, delimiter: str = '',
+                     continuation_token: str = '', start_after: str = '') -> ListStream:
         ...
 
     def head_object(self, bucket: str, key: str) -> TosObject:
@@ -147,10 +158,10 @@ class GetObjectOutput(object):
     content_range: str
     hash_crc64ecma: int
 
-    def read_all(self) -> bytes:
+    def read_all(self) -> Optional[bytes]:
         ...
 
-    def read(self) -> bytes:
+    def read(self) -> Optional[bytes]:
         ...
 
 

{tosnativeclient-1.0.2 → tosnativeclient-1.0.5}/src/list_stream.rs

@@ -3,7 +3,8 @@ use crate::tos_error::{map_error_from_string, map_tos_error};
 use crate::tos_model::ListObjectsResult;
 use arc_swap::ArcSwap;
 use async_channel::Receiver;
-use pyo3::{pyclass, pymethods, PyRef, PyRefMut, PyResult, Python};
+use pyo3::types::PyTuple;
+use pyo3::{pyclass, pymethods, Bound, IntoPyObject, PyRef, PyRefMut, PyResult, Python};
 use std::sync::atomic::{AtomicI8, Ordering};
 use std::sync::{Arc, RwLock};
 use tokio::runtime::Runtime;
@@ -12,7 +13,7 @@ use ve_tos_rust_sdk::error::TosError;
 use ve_tos_rust_sdk::object::{ListObjectsType2Input, ListObjectsType2Output};
 
 const DEFAULT_TASK_COUNT: usize = 5;
-#[pyclass(name = "ListStream")]
+#[pyclass(name = "ListStream", module = "tosnativeclient")]
 pub struct ListStream {
     client: Arc<InnerTosClient>,
     runtime: Arc<Runtime>,
@@ -26,6 +27,10 @@ pub struct ListStream {
     delimiter: String,
     #[pyo3(get)]
     max_keys: isize,
+    #[pyo3(get)]
+    continuation_token: String,
+    #[pyo3(get)]
+    start_after: String,
 }
 
 #[pymethods]
@@ -63,6 +68,22 @@ impl ListStream {
             }
         }
     }
+
+    pub fn current_prefix(&self) -> PyResult<Option<String>> {
+        let pg = self.paginator.read().unwrap();
+        match pg.as_ref() {
+            None => Ok(None),
+            Some(pg) => Ok(Some(pg.current_prefix())),
+        }
+    }
+
+    pub fn current_continuation_token(&self) -> PyResult<Option<String>> {
+        let pg = self.paginator.read().unwrap();
+        match pg.as_ref() {
+            None => Ok(None),
+            Some(pg) => Ok(Some(pg.current_continuation_token())),
+        }
+    }
 }
 
 impl ListStream {
@@ -73,6 +94,8 @@ impl ListStream {
         prefix: String,
         delimiter: String,
         max_keys: isize,
+        continuation_token: String,
+        start_after: String,
     ) -> Self {
         Self {
             client,
@@ -83,6 +106,8 @@ impl ListStream {
             prefix,
             delimiter,
             max_keys,
+            continuation_token,
+            start_after,
         }
     }
 
@@ -93,7 +118,12 @@ impl ListStream {
         input.set_prefix(self.prefix.as_str());
         input.set_max_keys(self.max_keys);
         input.set_delimiter(self.delimiter.as_str());
-
+        if self.continuation_token != "" {
+            input.set_continuation_token(self.continuation_token.as_str());
+        }
+        if self.start_after != "" {
+            input.set_start_after(self.start_after.as_str());
+        }
         py.allow_threads(|| {
             self.runtime.spawn(async move {
                 let mut need_break = false;
@@ -157,6 +187,7 @@ impl ListStream {
             is_end: ArcSwap::new(Arc::new(false)),
             last_err: ArcSwap::new(Arc::new(None)),
             current_prefix: ArcSwap::new(Arc::new(self.prefix.clone())),
+            current_continuation_token: ArcSwap::new(Arc::new(self.continuation_token.clone())),
             receiver,
         })
     }
@@ -195,6 +226,7 @@ pub(crate) struct Paginator {
     is_end: ArcSwap<bool>,
     last_err: ArcSwap<Option<TosError>>,
     current_prefix: ArcSwap<String>,
+    current_continuation_token: ArcSwap<String>,
     receiver: Receiver<(bool, Result<ListObjectsType2Output, TosError>)>,
 }
 
@@ -206,8 +238,11 @@ impl Paginator {
         Ok(!*self.is_end.load().as_ref())
     }
 
-    fn current_prefix(&self) -> Arc<String> {
-        self.current_prefix.load().clone()
+    fn current_prefix(&self) -> String {
+        self.current_prefix.load().to_string()
+    }
+    fn current_continuation_token(&self) -> String {
+        self.current_continuation_token.load().to_string()
     }
     async fn next_page(&self) -> Result<ListObjectsType2Output, TosError> {
         if let Some(e) = self.last_err.load().as_ref() {
@@ -236,6 +271,8 @@ impl Paginator {
                 Ok(output) => {
                     self.current_prefix
                         .store(Arc::new(output.prefix().to_string()));
+                    self.current_continuation_token
+                        .store(Arc::new(output.continuation_token().to_string()));
                     if is_end {
                         self.is_end.store(Arc::new(true));
                     }

{tosnativeclient-1.0.2 → tosnativeclient-1.0.5}/src/read_stream.rs

@@ -4,8 +4,8 @@ use crate::tos_model::TosObject;
 use async_channel::{Receiver, Sender};
 use bytes::Bytes;
 use futures_util::StreamExt;
-use pyo3::types::PyBytes;
-use pyo3::{pyclass, pymethods, Bound, PyRef, PyResult};
+use pyo3::types::{PyBytes, PyTuple};
+use pyo3::{pyclass, pymethods, Bound, IntoPyObject, PyRef, PyResult};
 use std::collections::LinkedList;
 use std::sync::atomic::{AtomicI8, AtomicIsize, Ordering};
 use std::sync::Arc;
@@ -24,7 +24,7 @@ const DEFAULT_PART_SIZE: isize = 8 * 1024 * 1024;
 const DEFAULT_FETCH_RETRY_COUNT: isize = 3;
 const DEFAULT_SHARED_PREFETCH_TASK_LIMIT: isize = 100;
 
-#[pyclass(name = "ReadStream")]
+#[pyclass(name = "ReadStream", module = "tosnativeclient")]
 pub struct ReadStream {
     object_fetcher: Arc<ObjectFetcher>,
     runtime: Arc<Runtime>,
@@ -117,6 +117,7 @@ impl ReadStream {
         Self {
             object_fetcher: Arc::new(ObjectFetcher {
                 client,
+                runtime: runtime.clone(),
                 closed: Arc::new(AtomicI8::new(0)),
                 pc,
                 fc: Mutex::new(fc),
@@ -149,6 +150,7 @@ impl ReadStream {
 
 struct ObjectFetcher {
     client: Arc<InnerTosClient>,
+    runtime: Arc<Runtime>,
     closed: Arc<AtomicI8>,
     pc: PrefetchConfig,
     fc: Mutex<FetchContext>,
@@ -295,7 +297,7 @@ impl ObjectFetcher {
                 if fc.has_tasks() {
                     fc.current = fc.pop_task()
                 } else {
-                    fc.current = self.start_fetch_task(fc);
+                    fc.current = self.start_fetch_task(fc, self.runtime.clone());
                 }
             }
             Some(current) => {
@@ -304,17 +306,17 @@ impl ObjectFetcher {
                     if fc.has_tasks() {
                         fc.current = fc.pop_task()
                     } else {
-                        fc.current = self.start_fetch_task(fc);
+                        fc.current = self.start_fetch_task(fc, self.runtime.clone());
                     }
                 } else if fc.exceed_sequential_read_threshold() {
                     if fc.can_add_task(self.pc.max_prefetch_tasks) {
-                        if let Some(next) = self.start_fetch_task(fc) {
+                        if let Some(next) = self.start_fetch_task(fc, self.runtime.clone()) {
                             fc.push_task(next, false);
                         }
                     } else if fc.can_steal_shared_task(self.pc.shared_prefetch_task_limit)
                         && fc.pcontext.try_steal_shared_prefetch_task()
                     {
-                        match self.start_fetch_task(fc) {
+                        match self.start_fetch_task(fc, self.runtime.clone()) {
                             Some(mut next) => {
                                 next.is_shared = true;
                                 fc.steal_shared_task();
@@ -330,14 +332,18 @@ impl ObjectFetcher {
         }
     }
 
-    fn start_fetch_task(&self, fc: &mut MutexGuard<'_, FetchContext>) -> Option<FetchTask> {
+    fn start_fetch_task(
+        &self,
+        fc: &mut MutexGuard<'_, FetchContext>,
+        runtime: Arc<Runtime>,
+    ) -> Option<FetchTask> {
         if fc.next_request_offset >= self.om.size {
             return None;
         }
 
         let task = self.new_fetch_task(fc);
         self.revise_next_request_offset_and_size(fc, task.size);
-        task.async_fetch(self.client.clone());
+        task.async_fetch(self.client.clone(), runtime);
         Some(task)
     }
 
@@ -660,9 +666,9 @@ impl FetchTask {
         self.start_offset + self.fetched_size.load(Ordering::Acquire)
     }
 
-    fn async_fetch(&self, client: Arc<InnerTosClient>) {
+    fn async_fetch(&self, client: Arc<InnerTosClient>, runtime: Arc<Runtime>) {
         let fetch_task_context = FetchTaskContext::new(self, client);
-        Handle::current().spawn(async move {
+        runtime.spawn(async move {
             fetch_task_context.fetch_from_server().await;
             fetch_task_context.chunk_queue.close();
         });

{tosnativeclient-1.0.2 → tosnativeclient-1.0.5}/src/tos_client.rs

@@ -5,7 +5,9 @@ use crate::tos_model::TosObject;
 use crate::write_stream::WriteStream;
 use async_trait::async_trait;
 use futures_util::future::BoxFuture;
-use pyo3::{pyclass, pymethods, PyRef, PyResult};
+use pyo3::prelude::PyDictMethods;
+use pyo3::types::{PyDict, PyTuple};
+use pyo3::{pyclass, pymethods, Bound, IntoPyObject, PyAny, PyObject, PyRef, PyRefMut, PyResult};
 use std::future::Future;
 use std::sync::atomic::{AtomicIsize, Ordering};
 use std::sync::Arc;
@@ -53,7 +55,7 @@ impl AsyncRuntime for TokioRuntime {
 pub(crate) type InnerTosClient =
     TosClientImpl<CommonCredentialsProvider<CommonCredentials>, CommonCredentials, TokioRuntime>;
 
-#[pyclass(name = "TosClient")]
+#[pyclass(name = "TosClient", module = "tosnativeclient")]
 pub struct TosClient {
     rclient: Arc<InnerTosClient>,
     wclient: Arc<InnerTosClient>,
@@ -75,6 +77,14 @@ pub struct TosClient {
     max_retry_count: isize,
     #[pyo3(get)]
     max_prefetch_tasks: isize,
+    #[pyo3(get)]
+    directives: String,
+    #[pyo3(get)]
+    directory: String,
+    #[pyo3(get)]
+    file_name_prefix: String,
+    #[pyo3(get)]
+    shared_prefetch_tasks: isize,
 }
 
 #[pymethods]
@@ -97,7 +107,11 @@ impl TosClient {
     ) -> PyResult<Self> {
         let mut _guard = None;
         if directives != "" {
-            _guard = Some(init_tracing_log(directives, directory, file_name_prefix));
+            _guard = Some(init_tracing_log(
+                directives.clone(),
+                directory.clone(),
+                file_name_prefix.clone(),
+            ));
         }
 
         let logical_cores = num_cpus::get();
@@ -142,16 +156,22 @@ impl TosClient {
             part_size,
             max_retry_count,
             max_prefetch_tasks,
+            directives,
+            directory,
+            file_name_prefix,
+            shared_prefetch_tasks,
         })
     }
 
-    #[pyo3(signature = (bucket, prefix=String::from(""), max_keys=1000, delimiter=String::from("")))]
+    #[pyo3(signature = (bucket, prefix=String::from(""), max_keys=1000, delimiter=String::from(""), continuation_token=String::from(""), start_after=String::from("")))]
     pub fn list_objects(
         &self,
         bucket: String,
         prefix: String,
         max_keys: isize,
         delimiter: String,
+        continuation_token: String,
+        start_after: String,
     ) -> ListStream {
         ListStream::new(
             self.rclient.clone(),
@@ -160,6 +180,8 @@ impl TosClient {
             prefix,
             delimiter,
             max_keys,
+            continuation_token,
+            start_after,
         )
     }
     pub fn head_object(slf: PyRef<'_, Self>, bucket: String, key: String) -> PyResult<TosObject> {
@@ -210,6 +232,24 @@ impl TosClient {
             })
         })
     }
+
+    pub fn __getnewargs__(slf: PyRef<'_, Self>) -> PyResult<Bound<'_, PyTuple>> {
+        let py = slf.py();
+        let state = [
+            slf.region.clone().into_pyobject(py)?.into_any(),
+            slf.endpoint.clone().into_pyobject(py)?.into_any(),
+            slf.ak.clone().into_pyobject(py)?.into_any(),
+            slf.sk.clone().into_pyobject(py)?.into_any(),
+            slf.part_size.into_pyobject(py)?.into_any(),
+            slf.max_retry_count.into_pyobject(py)?.into_any(),
+            slf.max_prefetch_tasks.into_pyobject(py)?.into_any(),
+            "".into_pyobject(py)?.into_any(),
+            "".into_pyobject(py)?.into_any(),
+            "".into_pyobject(py)?.into_any(),
+            slf.shared_prefetch_tasks.into_pyobject(py)?.into_any(),
+        ];
+        PyTuple::new(py, state)
+    }
 }
 
 pub(crate) struct SharedPrefetchContext {

{tosnativeclient-1.0.2 → tosnativeclient-1.0.5}/src/tos_error.rs

@@ -1,9 +1,10 @@
 use pyo3::exceptions::PyException;
-use pyo3::{create_exception, pyclass, pymethods, PyErr};
+use pyo3::types::PyTuple;
+use pyo3::{create_exception, pyclass, pymethods, Bound, IntoPyObject, PyErr, PyRef, PyResult};
 use std::error::Error;
 
-#[pyclass]
 #[derive(Clone)]
+#[pyclass(name = "TosError", module = "tosnativeclient")]
 pub struct TosError {
     #[pyo3(get)]
     message: String,
@@ -43,6 +44,17 @@ impl TosError {
             request_id,
         }
     }
+
+    pub fn __getnewargs__(slf: PyRef<'_, Self>) -> PyResult<Bound<'_, PyTuple>> {
+        let py = slf.py();
+        let state = [
+            slf.message.clone().into_pyobject(py)?.into_any(),
+            slf.status_code.clone().into_pyobject(py)?.into_any(),
+            slf.ec.clone().into_pyobject(py)?.into_any(),
+            slf.request_id.clone().into_pyobject(py)?.into_any(),
+        ];
+        PyTuple::new(py, state)
+    }
 }
 
 create_exception!(tosnativeclient, TosException, PyException);

{tosnativeclient-1.0.2 → tosnativeclient-1.0.5}/src/tos_model.rs

@@ -1,7 +1,7 @@
 use pyo3::pyclass;
 use ve_tos_rust_sdk::object::{HeadObjectOutput, ListObjectsType2Output};
 
-#[pyclass(name = "ListObjectsResult")]
+#[pyclass(name = "ListObjectsResult", module = "tosnativeclient")]
 pub struct ListObjectsResult {
     #[pyo3(get)]
     contents: Vec<TosObject>,
@@ -33,7 +33,7 @@ impl ListObjectsResult {
 }
 
 #[derive(Clone)]
-#[pyclass(name = "TosObject")]
+#[pyclass(name = "TosObject", module = "tosnativeclient")]
 pub struct TosObject {
     #[pyo3(get)]
     pub(crate) bucket: String,

{tosnativeclient-1.0.2 → tosnativeclient-1.0.5}/src/tos_raw_client.rs

@@ -2,8 +2,10 @@ use crate::tos_client::{InnerTosClient, TokioRuntime};
 use crate::tos_error::{map_error, map_tos_error};
 use bytes::Buf;
 use futures_util::StreamExt;
-use pyo3::{pyclass, pymethods, PyRef, PyRefMut, PyResult};
+use pyo3::types::PyTuple;
+use pyo3::{pyclass, pymethods, Bound, IntoPyObject, PyRef, PyRefMut, PyResult};
 use std::collections::HashMap;
+use std::sync::atomic::{AtomicI8, Ordering};
 use std::sync::Arc;
 use tokio::runtime;
 use tokio::runtime::Runtime;
@@ -11,7 +13,7 @@ use tokio::sync::Mutex;
 use ve_tos_rust_sdk::asynchronous::object::{ObjectAPI, ObjectContent};
 use ve_tos_rust_sdk::asynchronous::tos;
 
-#[pyclass(name = "TosRawClient")]
+#[pyclass(name = "TosRawClient", module = "tosnativeclient")]
 pub struct TosRawClient {
     client: Arc<InnerTosClient>,
     runtime: Arc<Runtime>,
@@ -158,7 +160,7 @@ impl TosRawClient {
                         content_range: output.content_range().to_string(),
                         hash_crc64ecma: output.hash_crc64ecma(),
                         output: Arc::new(Mutex::new(output)),
-                        runtime: runtime.clone(),
+                        runtime,
                     }),
                 }
             })
@@ -218,9 +220,24 @@ impl TosRawClient {
             })
         })
     }
+
+    pub fn __getnewargs__(slf: PyRef<'_, Self>) -> PyResult<Bound<'_, PyTuple>> {
+        let py = slf.py();
+        let state = [
+            slf.region.clone().into_pyobject(py)?.into_any(),
+            slf.endpoint.clone().into_pyobject(py)?.into_any(),
+            slf.ak.clone().into_pyobject(py)?.into_any(),
+            slf.sk.clone().into_pyobject(py)?.into_any(),
+            slf.connection_timeout.into_pyobject(py)?.into_any(),
+            slf.request_timeout.into_pyobject(py)?.into_any(),
+            slf.max_connections.into_pyobject(py)?.into_any(),
+            slf.max_retry_count.into_pyobject(py)?.into_any(),
+        ];
+        PyTuple::new(py, state)
+    }
 }
 
-#[pyclass(name = "HeadObjectInput")]
+#[pyclass(name = "HeadObjectInput", module = "tosnativeclient")]
 pub struct HeadObjectInput {
     #[pyo3(get, set)]
     pub(crate) bucket: String,
@@ -241,9 +258,18 @@ impl HeadObjectInput {
             version_id: version_id.to_string(),
         })
     }
+    pub fn __getnewargs__(slf: PyRef<'_, Self>) -> PyResult<Bound<'_, PyTuple>> {
+        let py = slf.py();
+        let state = [
+            slf.bucket.clone().into_pyobject(py)?.into_any(),
+            slf.key.clone().into_pyobject(py)?.into_any(),
+            slf.version_id.clone().into_pyobject(py)?.into_any(),
+        ];
+        PyTuple::new(py, state)
+    }
 }
 
-#[pyclass(name = "HeadObjectOutput")]
+#[pyclass(name = "HeadObjectOutput", module = "tosnativeclient")]
 pub struct HeadObjectOutput {
     #[pyo3(get)]
     pub(crate) request_id: String,
@@ -261,7 +287,7 @@ pub struct HeadObjectOutput {
     pub(crate) hash_crc64ecma: u64,
 }
 
-#[pyclass(name = "DeleteObjectInput")]
+#[pyclass(name = "DeleteObjectInput", module = "tosnativeclient")]
 pub struct DeleteObjectInput {
     #[pyo3(get, set)]
     pub(crate) bucket: String,
@@ -282,9 +308,18 @@ impl DeleteObjectInput {
             version_id: version_id.to_string(),
         })
     }
+    pub fn __getnewargs__(slf: PyRef<'_, Self>) -> PyResult<Bound<'_, PyTuple>> {
+        let py = slf.py();
+        let state = [
+            slf.bucket.clone().into_pyobject(py)?.into_any(),
+            slf.key.clone().into_pyobject(py)?.into_any(),
+            slf.version_id.clone().into_pyobject(py)?.into_any(),
+        ];
+        PyTuple::new(py, state)
+    }
 }
 
-#[pyclass(name = "DeleteObjectOutput")]
+#[pyclass(name = "DeleteObjectOutput", module = "tosnativeclient")]
 pub struct DeleteObjectOutput {
     #[pyo3(get)]
     pub(crate) request_id: String,
@@ -297,7 +332,7 @@ pub struct DeleteObjectOutput {
     #[pyo3(get)]
     pub(crate) version_id: String,
 }
-#[pyclass(name = "GetObjectInput")]
+#[pyclass(name = "GetObjectInput", module = "tosnativeclient")]
 pub struct GetObjectInput {
     #[pyo3(get, set)]
     pub(crate) bucket: String,
@@ -321,9 +356,19 @@ impl GetObjectInput {
             range: range.to_string(),
         })
     }
+    pub fn __getnewargs__(slf: PyRef<'_, Self>) -> PyResult<Bound<'_, PyTuple>> {
+        let py = slf.py();
+        let state = [
+            slf.bucket.clone().into_pyobject(py)?.into_any(),
+            slf.key.clone().into_pyobject(py)?.into_any(),
+            slf.version_id.clone().into_pyobject(py)?.into_any(),
+            slf.range.clone().into_pyobject(py)?.into_any(),
+        ];
+        PyTuple::new(py, state)
+    }
 }
 
-#[pyclass(name = "GetObjectOutput")]
+#[pyclass(name = "GetObjectOutput", module = "tosnativeclient")]
 pub struct GetObjectOutput {
     #[pyo3(get)]
     pub(crate) request_id: String,
@@ -350,30 +395,34 @@ impl GetObjectOutput {
     pub fn read_all(slf: PyRefMut<'_, Self>) -> PyResult<Option<Vec<u8>>> {
         let runtime = slf.runtime.clone();
         let output = slf.output.clone();
-        runtime.block_on(async move {
-            match output.lock().await.read_all().await {
-                Err(ex) => Err(map_tos_error(ex)),
-                Ok(buf) => Ok(Some(buf)),
-            }
+        slf.py().allow_threads(|| {
+            runtime.block_on(async move {
+                match output.lock().await.read_all().await {
+                    Err(ex) => Err(map_tos_error(ex)),
+                    Ok(buf) => Ok(Some(buf)),
+                }
+            })
         })
     }
 
     pub fn read(slf: PyRefMut<'_, Self>) -> PyResult<Option<Vec<u8>>> {
         let runtime = slf.runtime.clone();
         let output = slf.output.clone();
-        runtime.block_on(async move {
-            match output.lock().await.next().await {
-                None => Ok(None),
-                Some(result) => match result {
-                    Err(ex) => Err(map_error(ex)),
-                    Ok(buf) => Ok(Some(buf.chunk().to_vec())),
-                },
-            }
+        slf.py().allow_threads(|| {
+            runtime.block_on(async move {
+                match output.lock().await.next().await {
+                    None => Ok(None),
+                    Some(result) => match result {
+                        Err(ex) => Err(map_error(ex)),
+                        Ok(buf) => Ok(Some(buf.chunk().to_vec())),
+                    },
+                }
+            })
         })
     }
 }
 
-#[pyclass(name = "PutObjectFromBufferInput")]
+#[pyclass(name = "PutObjectFromBufferInput", module = "tosnativeclient")]
 pub struct PutObjectFromBufferInput {
     #[pyo3(get, set)]
     pub(crate) bucket: String,
@@ -394,9 +443,18 @@ impl PutObjectFromBufferInput {
             content: content.to_vec(),
         })
     }
+    pub fn __getnewargs__(slf: PyRef<'_, Self>) -> PyResult<Bound<'_, PyTuple>> {
+        let py = slf.py();
+        let state = [
+            slf.bucket.clone().into_pyobject(py)?.into_any(),
+            slf.key.clone().into_pyobject(py)?.into_any(),
+            slf.content.clone().into_pyobject(py)?.into_any(),
+        ];
+        PyTuple::new(py, state)
+    }
 }
 
-#[pyclass(name = "PutObjectFromFileInput")]
+#[pyclass(name = "PutObjectFromFileInput", module = "tosnativeclient")]
 pub struct PutObjectFromFileInput {
     #[pyo3(get, set)]
     pub(crate) bucket: String,
@@ -417,9 +475,18 @@ impl PutObjectFromFileInput {
             file_path: file_path.to_string(),
         })
     }
+    pub fn __getnewargs__(slf: PyRef<'_, Self>) -> PyResult<Bound<'_, PyTuple>> {
+        let py = slf.py();
+        let state = [
+            slf.bucket.clone().into_pyobject(py)?.into_any(),
+            slf.key.clone().into_pyobject(py)?.into_any(),
+            slf.file_path.clone().into_pyobject(py)?.into_any(),
+        ];
+        PyTuple::new(py, state)
+    }
 }
 
-#[pyclass(name = "PutObjectOutput")]
+#[pyclass(name = "PutObjectOutput", module = "tosnativeclient")]
 pub struct PutObjectOutput {
     #[pyo3(get)]
     pub(crate) request_id: String,

{tosnativeclient-1.0.2 → tosnativeclient-1.0.5}/src/write_stream.rs

@@ -2,7 +2,8 @@ use crate::tos_client::InnerTosClient;
 use crate::tos_error::map_tos_error;
 use async_channel::{Receiver, Sender};
 use futures_util::future::join_all;
-use pyo3::{pyclass, pymethods, PyRefMut, PyResult};
+use pyo3::types::PyTuple;
+use pyo3::{pyclass, pymethods, Bound, IntoPyObject, PyRef, PyRefMut, PyResult};
 use std::collections::HashMap;
 use std::sync::atomic::{AtomicI8, AtomicIsize, Ordering};
 use std::sync::Arc;
@@ -29,7 +30,7 @@ const MAX_PART_NUMBER: isize = 10000;
 const OTHER_MU_KICK_OFF: i8 = 1;
 const RELEASE_MU_KICK_OFF: i8 = 2;
 
-#[pyclass(name = "WriteStream")]
+#[pyclass(name = "WriteStream", module = "tosnativeclient")]
 pub struct WriteStream {
     object_writer: Arc<ObjectWriter>,
     runtime: Arc<Runtime>,
@@ -93,8 +94,15 @@ impl WriteStream {
         let _key = key.clone();
         let _storage_class = storage_class.clone();
 
-        let object_writer =
-            ObjectWriter::new(client, _bucket, _key, _storage_class, part_size).await?;
+        let object_writer = ObjectWriter::new(
+            client,
+            runtime.clone(),
+            _bucket,
+            _key,
+            _storage_class,
+            part_size,
+        )
+        .await?;
         Ok(Self {
             object_writer: Arc::new(object_writer),
             runtime,
@@ -134,6 +142,7 @@ struct ObjectWriter {
 impl ObjectWriter {
     async fn new(
         client: Arc<InnerTosClient>,
+        runtime: Arc<Runtime>,
         bucket: String,
         key: String,
         storage_class: Option<String>,
@@ -144,7 +153,15 @@ impl ObjectWriter {
             key,
             storage_class,
         });
-        let ou = ObjectUploader::new(client, wp, part_size, 0, Arc::new(AtomicI8::new(0))).await?;
+        let ou = ObjectUploader::new(
+            client,
+            runtime,
+            wp,
+            part_size,
+            0,
+            Arc::new(AtomicI8::new(0)),
+        )
+        .await?;
         Ok(Self {
             ctx: Mutex::new((ou, 0)),
             closed: Arc::new(AtomicI8::new(0)),
@@ -263,6 +280,7 @@ struct WriteParam {
 struct ObjectUploader {
     next_write_offset: isize,
     uc: Arc<UploadContext>,
+    runtime: Arc<Runtime>,
     current: Option<Part>,
     part_size: isize,
     max_size: isize,
@@ -277,6 +295,7 @@ struct ObjectUploader {
 impl ObjectUploader {
     async fn new(
         client: Arc<InnerTosClient>,
+        runtime: Arc<Runtime>,
         wp: Arc<WriteParam>,
         part_size: isize,
         next_write_offset: isize,
@@ -293,6 +312,7 @@ impl ObjectUploader {
         let mut ou = Self {
             next_write_offset,
             uc: Arc::new(UploadContext::new(created, wp, client)),
+            runtime,
             current: None,
             part_size,
             max_size,
@@ -474,7 +494,7 @@ impl ObjectUploader {
         let dp = self.dp.clone();
         let st = self.st.clone();
         let uc = self.uc.clone();
-        self.wait_dispatch = Some(Handle::current().spawn(async move {
+        self.wait_dispatch = Some(self.runtime.spawn(async move {
             loop {
                 match dp.pull().await {
                     None => return,
@@ -499,12 +519,13 @@ impl ObjectUploader {
     }
 
     async fn execute(&mut self) {
+        let runtime = self.runtime.clone();
         let dp = self.dp.clone();
         let st = self.st.clone();
         let ta = self.ta.clone();
         let uc = self.uc.clone();
         let mu_ctx = self.mu_ctx.clone();
-        self.wait_execute = Some(Handle::current().spawn(async move {
+        self.wait_execute = Some(self.runtime.spawn(async move {
             let mut wait_async_uploads = Vec::with_capacity(16);
             loop {
                 match st.pull().await {
@@ -518,6 +539,7 @@ impl ObjectUploader {
                         wait_async_uploads.push(
                             uc.clone()
                                 .async_upload(
+                                    runtime.clone(),
                                     dp.clone(),
                                     st.clone(),
                                     ta.clone(),
@@ -589,6 +611,7 @@ impl UploadContext {
 
     async fn async_upload(
         self: Arc<Self>,
+        runtime: Arc<Runtime>,
         dp: Arc<Dispatcher>,
         st: Arc<Store>,
         ta: Arc<TokenAcquirer>,
@@ -596,7 +619,7 @@ impl UploadContext {
         mut si: StoreItem,
     ) -> JoinHandle<()> {
         let aborted = self.aborted.clone();
-        Handle::current().spawn(async move {
+        runtime.spawn(async move {
             let _ = ta.acquire().await;
             if aborted.load(Ordering::Acquire) == 1 {
                 si.release();