tosnativeclient 1.0.1__tar.gz → 1.0.4__tar.gz

{tosnativeclient-1.0.1 → tosnativeclient-1.0.4}/Cargo.lock

@@ -472,6 +472,12 @@ version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
 
+[[package]]
+name = "hermit-abi"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c"
+
 [[package]]
 name = "hex"
 version = "0.4.3"
@@ -859,6 +865,16 @@ dependencies = [
  "autocfg",
 ]
 
+[[package]]
+name = "num_cpus"
+version = "1.17.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b"
+dependencies = [
+ "hermit-abi",
+ "libc",
+]
+
 [[package]]
 name = "object"
 version = "0.36.7"
@@ -1505,13 +1521,14 @@ dependencies = [
 
 [[package]]
 name = "tosnativeclient"
-version = "1.0.1"
+version = "1.0.4"
 dependencies = [
  "arc-swap",
  "async-channel",
  "async-trait",
  "bytes",
  "futures-util",
+ "num_cpus",
  "pyo3",
  "tokio",
  "tracing",

{tosnativeclient-1.0.1 → tosnativeclient-1.0.4}/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "tosnativeclient"
-version = "1.0.1"
+version = "1.0.4"
 edition = "2021"
 publish = false
 authors = ["xiangshijian"]
@@ -23,3 +23,4 @@ async-trait = "0.1.88"
 futures-util = "0.3.30"
 async-channel = "2.3.1"
 tracing = { version = "0.1.41", features = ["log"] }
+num_cpus = "1.17.0"

{tosnativeclient-1.0.1 → tosnativeclient-1.0.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tosnativeclient
-Version: 1.0.1
+Version: 1.0.4
 Classifier: Programming Language :: Rust
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy

tosnativeclient-1.0.4/python/tosnativeclient/__init__.py

@@ -0,0 +1,25 @@
+from .tosnativeclient import TosClient, ListStream, ListObjectsResult, TosObject, ReadStream, WriteStream, TosError, \
+    TosException, TosRawClient, \
+    HeadObjectInput, HeadObjectOutput, GetObjectOutput, DeleteObjectInput, DeleteObjectOutput, GetObjectInput, \
+    PutObjectFromBufferInput, PutObjectFromFileInput, PutObjectOutput
+
+__all__ = [
+    'TosError',
+    'TosException',
+    'TosClient',
+    'ListStream',
+    'ListObjectsResult',
+    'TosObject',
+    'ReadStream',
+    'WriteStream',
+    'TosRawClient',
+    'HeadObjectInput',
+    'HeadObjectOutput',
+    'DeleteObjectInput',
+    'DeleteObjectOutput',
+    'GetObjectInput',
+    'GetObjectOutput',
+    'PutObjectFromBufferInput',
+    'PutObjectFromFileInput',
+    'PutObjectOutput'
+]

{tosnativeclient-1.0.1 → tosnativeclient-1.0.4}/python/tosnativeclient/tosnativeclient.pyi

@@ -3,31 +3,27 @@ from typing import List, Dict
 from typing import Optional
 
 
-class TosClient(object):
-    region: str
-    endpoint: str
-    ak: str
-    sk: str
-    part_size: int
-    max_retry_count: int
-    max_prefetch_tasks: int
+class TosError(object):
+    message: str
+    status_code: Optional[int]
+    ec: str
+    request_id: str
 
-    def __init__(self, region: str, endpoint: str, ak: str = '', sk: str = '', part_size: int = 8388608,
-                 max_retry_count: int = 3, max_prefetch_tasks: int = 3, directives: str = '', directory: str = '',
-                 file_name_prefix: str = '', shared_prefetch_tasks: int = 20):
-        ...
 
-    def list_objects(self, bucket: str, prefix: str = '', max_keys: int = 1000, delimiter: str = '') -> ListStream:
-        ...
+class TosException(Exception):
+    args: List[TosError]
 
-    def head_object(self, bucket: str, key: str) -> TosObject:
-        ...
 
-    def get_object(self, bucket: str, key: str, etag: str, size: int) -> ReadStream:
-        ...
+class TosObject(object):
+    bucket: str
+    key: str
+    size: int
+    etag: str
 
-    def put_object(self, bucket: str, key: str, storage_class: str = '') -> WriteStream:
-        ...
+
+class ListObjectsResult(object):
+    contents: List[TosObject]
+    common_prefixes: List[str]
 
 
 class ListStream(object):
@@ -43,25 +39,13 @@ class ListStream(object):
     def close(self) -> None: ...
 
 
-class ListObjectsResult(object):
-    contents: List[TosObject]
-    common_prefixes: List[str]
-
-
-class TosObject(object):
-    bucket: str
-    key: str
-    size: int
-    etag: str
-
-
 class ReadStream(object):
     bucket: str
     key: str
     size: int
     etag: str
 
-    def read(self, offset: int, length: int) -> bytes:
+    def read(self, offset: int, length: int) -> Optional[bytes]:
         ...
 
     def close(self) -> None:
@@ -80,44 +64,34 @@ class WriteStream(object):
         ...
 
 
-class TosError(object):
-    message: str
-    status_code: Optional[int]
-    ec: str
-    request_id: str
-
-
-class TosException(Exception):
-    args: List[TosError]
-
-
-class TosRawClient(object):
+class TosClient(object):
     region: str
     endpoint: str
     ak: str
     sk: str
-    connection_timeout: int
-    request_timeout: int
-    max_connections: int
+    part_size: int
     max_retry_count: int
+    max_prefetch_tasks: int
+    directives: str
+    directory: str
+    file_name_prefix: str
+    shared_prefetch_tasks: int
 
-    def __init__(self, region: str, endpoint: str, ak: str = '', sk: str = '', connection_timeout: int = 10000,
-                 request_timeout: int = 120000, max_connections: int = 1024, max_retry_count: int = 3):
-        ...
-
-    def head_object(self, input: HeadObjectInput) -> HeadObjectOutput:
+    def __init__(self, region: str, endpoint: str, ak: str = '', sk: str = '', part_size: int = 8388608,
+                 max_retry_count: int = 3, max_prefetch_tasks: int = 3, directives: str = '', directory: str = '',
+                 file_name_prefix: str = '', shared_prefetch_tasks: int = 20):
         ...
 
-    def delete_object(self, input: DeleteObjectInput) -> DeleteObjectOutput:
+    def list_objects(self, bucket: str, prefix: str = '', max_keys: int = 1000, delimiter: str = '') -> ListStream:
         ...
 
-    def get_object(self, input: GetObjectInput) -> GetObjectOutput:
+    def head_object(self, bucket: str, key: str) -> TosObject:
         ...
 
-    def put_object_from_buffer(self, input: PutObjectFromBufferInput) -> PutObjectOutput:
+    def get_object(self, bucket: str, key: str, etag: str, size: int) -> ReadStream:
         ...
 
-    def put_object_from_file(self, input: PutObjectFromFileInput) -> PutObjectOutput:
+    def put_object(self, bucket: str, key: str, storage_class: Optional[str] = '') -> WriteStream:
         ...
 
 
@@ -177,10 +151,10 @@ class GetObjectOutput(object):
     content_range: str
     hash_crc64ecma: int
 
-    def read_all(self) -> bytes:
+    def read_all(self) -> Optional[bytes]:
         ...
 
-    def read(self) -> bytes:
+    def read(self) -> Optional[bytes]:
         ...
 
 
@@ -209,3 +183,33 @@ class PutObjectOutput(object):
     etag: str
     version_id: str
     hash_crc64ecma: int
+
+
+class TosRawClient(object):
+    region: str
+    endpoint: str
+    ak: str
+    sk: str
+    connection_timeout: int
+    request_timeout: int
+    max_connections: int
+    max_retry_count: int
+
+    def __init__(self, region: str, endpoint: str, ak: str = '', sk: str = '', connection_timeout: int = 10000,
+                 request_timeout: int = 120000, max_connections: int = 1024, max_retry_count: int = 3):
+        ...
+
+    def head_object(self, input: HeadObjectInput) -> HeadObjectOutput:
+        ...
+
+    def delete_object(self, input: DeleteObjectInput) -> DeleteObjectOutput:
+        ...
+
+    def get_object(self, input: GetObjectInput) -> GetObjectOutput:
+        ...
+
+    def put_object_from_buffer(self, input: PutObjectFromBufferInput) -> PutObjectOutput:
+        ...
+
+    def put_object_from_file(self, input: PutObjectFromFileInput) -> PutObjectOutput:
+        ...

{tosnativeclient-1.0.1 → tosnativeclient-1.0.4}/src/list_stream.rs

@@ -3,7 +3,8 @@ use crate::tos_error::{map_error_from_string, map_tos_error};
 use crate::tos_model::ListObjectsResult;
 use arc_swap::ArcSwap;
 use async_channel::Receiver;
-use pyo3::{pyclass, pymethods, PyRef, PyRefMut, PyResult, Python};
+use pyo3::types::PyTuple;
+use pyo3::{pyclass, pymethods, Bound, IntoPyObject, PyRef, PyRefMut, PyResult, Python};
 use std::sync::atomic::{AtomicI8, Ordering};
 use std::sync::{Arc, RwLock};
 use tokio::runtime::Runtime;
@@ -12,7 +13,7 @@ use ve_tos_rust_sdk::error::TosError;
 use ve_tos_rust_sdk::object::{ListObjectsType2Input, ListObjectsType2Output};
 
 const DEFAULT_TASK_COUNT: usize = 5;
-#[pyclass(name = "ListStream")]
+#[pyclass(name = "ListStream", module = "tosnativeclient")]
 pub struct ListStream {
     client: Arc<InnerTosClient>,
     runtime: Arc<Runtime>,
@@ -46,14 +47,11 @@ impl ListStream {
             return Err(map_error_from_string("ListStream is closed"));
         }
 
-        {
-            let mut pg = slf.paginator.write().unwrap();
-            if pg.is_none() {
-                *pg = slf.list_background(slf.py());
-            }
+        let mut pg = slf.paginator.write().unwrap();
+        if pg.is_none() {
+            *pg = slf.list_background(slf.py());
         }
-
-        slf.next_page(slf.paginator.read().unwrap().as_ref(), slf.py())
+        slf.next_page(pg.as_ref(), slf.py())
     }
 
     pub fn close(&self) {
@@ -66,6 +64,17 @@ impl ListStream {
             }
         }
     }
+
+    pub fn __getnewargs__(slf: PyRef<'_, Self>) -> PyResult<Bound<'_, PyTuple>> {
+        let py = slf.py();
+        let state = [
+            slf.bucket.clone().into_pyobject(py)?.into_any(),
+            slf.prefix.clone().into_pyobject(py)?.into_any(),
+            slf.delimiter.clone().into_pyobject(py)?.into_any(),
+            slf.max_keys.clone().into_pyobject(py)?.into_any(),
+        ];
+        PyTuple::new(py, state)
+    }
 }
 
 impl ListStream {
@@ -100,67 +109,54 @@ impl ListStream {
         py.allow_threads(|| {
             self.runtime.spawn(async move {
                 let mut need_break = false;
-                if input.delimiter() == "/" {
-                    let mut prefixes = Vec::with_capacity(16);
-                    let mut last_page_end = false;
+                if input.delimiter() == "" {
                     loop {
-                        if last_page_end {
-                            if prefixes.is_empty() {
-                                let _ = sender
-                                    .send((
-                                        true,
-                                        Err(TosError::TosClientError {
-                                            message: "invalid status error".to_string(),
-                                            cause: None,
-                                        }),
-                                    ))
-                                    .await;
-                                break;
-                            }
-
-                            let prefix = prefixes.remove(0);
-                            input.set_prefix(prefix);
-                            input.set_start_after("");
-                            input.set_continuation_token("");
-                            last_page_end = false;
-                        }
                         let result = client.list_objects_type2(&input).await;
                         if let Ok(ref o) = result {
                             if o.is_truncated() {
                                 input.set_continuation_token(o.next_continuation_token());
                             } else {
-                                last_page_end = true;
-                            }
-
-                            for cp in o.common_prefixes() {
-                                prefixes.push(cp.prefix().to_string());
+                                need_break = true;
                             }
-                            need_break = last_page_end && prefixes.is_empty();
                         } else {
                             need_break = true;
                         }
-
                         if let Err(_) = sender.send((need_break, result)).await {
-                            need_break = true;
+                            need_break = true
                         }
                         if need_break {
                             break;
                         }
                     }
                 } else {
+                    let mut prefixes = Vec::with_capacity(16);
+                    let mut last_page_end = false;
                     loop {
+                        if last_page_end {
+                            let prefix = prefixes.remove(0);
+                            input.set_prefix(prefix);
+                            input.set_start_after("");
+                            input.set_continuation_token("");
+                            last_page_end = false;
+                        }
                         let result = client.list_objects_type2(&input).await;
                         if let Ok(ref o) = result {
                             if o.is_truncated() {
                                 input.set_continuation_token(o.next_continuation_token());
                             } else {
-                                need_break = true;
+                                last_page_end = true;
+                            }
+
+                            for cp in o.common_prefixes() {
+                                prefixes.push(cp.prefix().to_string());
                             }
+                            need_break = last_page_end && prefixes.is_empty();
                         } else {
                             need_break = true;
                         }
+
                         if let Err(_) = sender.send((need_break, result)).await {
-                            need_break = true
+                            need_break = true;
                         }
                         if need_break {
                             break;
@@ -169,7 +165,6 @@ impl ListStream {
                 }
             });
         });
-
         Some(Paginator {
             is_end: ArcSwap::new(Arc::new(false)),
             last_err: ArcSwap::new(Arc::new(None)),

{tosnativeclient-1.0.1 → tosnativeclient-1.0.4}/src/read_stream.rs

@@ -4,8 +4,8 @@ use crate::tos_model::TosObject;
 use async_channel::{Receiver, Sender};
 use bytes::Bytes;
 use futures_util::StreamExt;
-use pyo3::types::PyBytes;
-use pyo3::{pyclass, pymethods, Bound, PyRef, PyResult};
+use pyo3::types::{PyBytes, PyTuple};
+use pyo3::{pyclass, pymethods, Bound, IntoPyObject, PyRef, PyResult};
 use std::collections::LinkedList;
 use std::sync::atomic::{AtomicI8, AtomicIsize, Ordering};
 use std::sync::Arc;
@@ -24,7 +24,7 @@ const DEFAULT_PART_SIZE: isize = 8 * 1024 * 1024;
 const DEFAULT_FETCH_RETRY_COUNT: isize = 3;
 const DEFAULT_SHARED_PREFETCH_TASK_LIMIT: isize = 100;
 
-#[pyclass(name = "ReadStream")]
+#[pyclass(name = "ReadStream", module = "tosnativeclient")]
 pub struct ReadStream {
     object_fetcher: Arc<ObjectFetcher>,
     runtime: Arc<Runtime>,
@@ -66,6 +66,17 @@ impl ReadStream {
         slf.py()
             .allow_threads(|| runtime.block_on(async move { fetcher.close().await }))
     }
+
+    pub fn __getnewargs__(slf: PyRef<'_, Self>) -> PyResult<Bound<'_, PyTuple>> {
+        let py = slf.py();
+        let state = [
+            slf.bucket.clone().into_pyobject(py)?.into_any(),
+            slf.key.clone().into_pyobject(py)?.into_any(),
+            slf.etag.clone().into_pyobject(py)?.into_any(),
+            slf.size.clone().into_pyobject(py)?.into_any(),
+        ];
+        PyTuple::new(py, state)
+    }
 }
 
 impl ReadStream {

{tosnativeclient-1.0.1 → tosnativeclient-1.0.4}/src/tos_client.rs

@@ -5,13 +5,14 @@ use crate::tos_model::TosObject;
 use crate::write_stream::WriteStream;
 use async_trait::async_trait;
 use futures_util::future::BoxFuture;
-use pyo3::{pyclass, pymethods, PyRef, PyResult};
+use pyo3::prelude::PyDictMethods;
+use pyo3::types::{PyDict, PyTuple};
+use pyo3::{pyclass, pymethods, Bound, IntoPyObject, PyAny, PyObject, PyRef, PyRefMut, PyResult};
 use std::future::Future;
 use std::sync::atomic::{AtomicIsize, Ordering};
 use std::sync::Arc;
 use std::time::Duration;
-use tokio::runtime;
-use tokio::runtime::{Handle, Runtime};
+use tokio::runtime::{Builder, Handle, Runtime};
 use tracing_appender::non_blocking::WorkerGuard;
 use ve_tos_rust_sdk::asynchronous::object::ObjectAPI;
 use ve_tos_rust_sdk::asynchronous::tos;
@@ -54,7 +55,7 @@ impl AsyncRuntime for TokioRuntime {
 pub(crate) type InnerTosClient =
     TosClientImpl<CommonCredentialsProvider<CommonCredentials>, CommonCredentials, TokioRuntime>;
 
-#[pyclass(name = "TosClient")]
+#[pyclass(name = "TosClient", module = "tosnativeclient")]
 pub struct TosClient {
     rclient: Arc<InnerTosClient>,
     wclient: Arc<InnerTosClient>,
@@ -76,6 +77,14 @@ pub struct TosClient {
     max_retry_count: isize,
     #[pyo3(get)]
     max_prefetch_tasks: isize,
+    #[pyo3(get)]
+    directives: String,
+    #[pyo3(get)]
+    directory: String,
+    #[pyo3(get)]
+    file_name_prefix: String,
+    #[pyo3(get)]
+    shared_prefetch_tasks: isize,
 }
 
 #[pymethods]
@@ -98,10 +107,19 @@ impl TosClient {
     ) -> PyResult<Self> {
         let mut _guard = None;
         if directives != "" {
-            _guard = Some(init_tracing_log(directives, directory, file_name_prefix));
+            _guard = Some(init_tracing_log(
+                directives.clone(),
+                directory.clone(),
+                file_name_prefix.clone(),
+            ));
         }
 
-        let runtime = Arc::new(runtime::Builder::new_multi_thread().enable_all().build()?);
+        let logical_cores = num_cpus::get();
+        let mut builder = Builder::new_multi_thread();
+        if logical_cores > 0 {
+            builder.worker_threads(logical_cores);
+        }
+        let runtime = Arc::new(builder.enable_all().build()?);
         let mut clients = Vec::with_capacity(2);
         for _ in 0..2 {
             match tos::builder()
@@ -138,6 +156,10 @@ impl TosClient {
             part_size,
             max_retry_count,
             max_prefetch_tasks,
+            directives,
+            directory,
+            file_name_prefix,
+            shared_prefetch_tasks,
         })
     }
 
@@ -188,22 +210,41 @@ impl TosClient {
 
     #[pyo3(signature = (bucket, key, storage_class=None))]
     pub fn put_object(
-        &self,
+        slf: PyRef<'_, Self>,
         bucket: String,
         key: String,
         storage_class: Option<String>,
     ) -> PyResult<WriteStream> {
-        match WriteStream::new(
-            self.wclient.clone(),
-            self.runtime.clone(),
-            bucket,
-            key,
-            storage_class,
-            self.part_size,
-        ) {
-            Err(ex) => Err(map_tos_error(ex)),
-            Ok(ws) => Ok(ws),
-        }
+        let client = slf.wclient.clone();
+        let runtime = slf.runtime.clone();
+        let part_size = slf.part_size;
+        slf.py().allow_threads(|| {
+            runtime.clone().block_on(async move {
+                match WriteStream::new(client, runtime, bucket, key, storage_class, part_size).await
+                {
+                    Err(ex) => Err(map_tos_error(ex)),
+                    Ok(ws) => Ok(ws),
+                }
+            })
+        })
+    }
+
+    pub fn __getnewargs__(slf: PyRef<'_, Self>) -> PyResult<Bound<'_, PyTuple>> {
+        let py = slf.py();
+        let state = [
+            slf.region.clone().into_pyobject(py)?.into_any(),
+            slf.endpoint.clone().into_pyobject(py)?.into_any(),
+            slf.ak.clone().into_pyobject(py)?.into_any(),
+            slf.sk.clone().into_pyobject(py)?.into_any(),
+            slf.part_size.into_pyobject(py)?.into_any(),
+            slf.max_retry_count.into_pyobject(py)?.into_any(),
+            slf.max_prefetch_tasks.into_pyobject(py)?.into_any(),
+            "".into_pyobject(py)?.into_any(),
+            "".into_pyobject(py)?.into_any(),
+            "".into_pyobject(py)?.into_any(),
+            slf.shared_prefetch_tasks.into_pyobject(py)?.into_any(),
+        ];
+        PyTuple::new(py, state)
     }
 }
 

{tosnativeclient-1.0.1 → tosnativeclient-1.0.4}/src/tos_error.rs

@@ -1,9 +1,10 @@
 use pyo3::exceptions::PyException;
-use pyo3::{create_exception, pyclass, pymethods, PyErr};
+use pyo3::types::PyTuple;
+use pyo3::{create_exception, pyclass, pymethods, Bound, IntoPyObject, PyErr, PyRef, PyResult};
 use std::error::Error;
 
-#[pyclass]
 #[derive(Clone)]
+#[pyclass(name = "TosError", module = "tosnativeclient")]
 pub struct TosError {
     #[pyo3(get)]
     message: String,
@@ -43,6 +44,17 @@ impl TosError {
             request_id,
         }
     }
+
+    pub fn __getnewargs__(slf: PyRef<'_, Self>) -> PyResult<Bound<'_, PyTuple>> {
+        let py = slf.py();
+        let state = [
+            slf.message.clone().into_pyobject(py)?.into_any(),
+            slf.status_code.clone().into_pyobject(py)?.into_any(),
+            slf.ec.clone().into_pyobject(py)?.into_any(),
+            slf.request_id.clone().into_pyobject(py)?.into_any(),
+        ];
+        PyTuple::new(py, state)
+    }
 }
 
 create_exception!(tosnativeclient, TosException, PyException);

{tosnativeclient-1.0.1 → tosnativeclient-1.0.4}/src/tos_model.rs

@@ -1,7 +1,7 @@
 use pyo3::pyclass;
 use ve_tos_rust_sdk::object::{HeadObjectOutput, ListObjectsType2Output};
 
-#[pyclass(name = "ListObjectsResult")]
+#[pyclass(name = "ListObjectsResult", module = "tosnativeclient")]
 pub struct ListObjectsResult {
     #[pyo3(get)]
     contents: Vec<TosObject>,
@@ -33,7 +33,7 @@ impl ListObjectsResult {
 }
 
 #[derive(Clone)]
-#[pyclass(name = "TosObject")]
+#[pyclass(name = "TosObject", module = "tosnativeclient")]
 pub struct TosObject {
     #[pyo3(get)]
     pub(crate) bucket: String,

{tosnativeclient-1.0.1 → tosnativeclient-1.0.4}/src/tos_raw_client.rs

@@ -2,8 +2,10 @@ use crate::tos_client::{InnerTosClient, TokioRuntime};
 use crate::tos_error::{map_error, map_tos_error};
 use bytes::Buf;
 use futures_util::StreamExt;
-use pyo3::{pyclass, pymethods, PyRef, PyRefMut, PyResult};
+use pyo3::types::PyTuple;
+use pyo3::{pyclass, pymethods, Bound, IntoPyObject, PyRef, PyRefMut, PyResult};
 use std::collections::HashMap;
+use std::sync::atomic::{AtomicI8, Ordering};
 use std::sync::Arc;
 use tokio::runtime;
 use tokio::runtime::Runtime;
@@ -11,7 +13,7 @@ use tokio::sync::Mutex;
 use ve_tos_rust_sdk::asynchronous::object::{ObjectAPI, ObjectContent};
 use ve_tos_rust_sdk::asynchronous::tos;
 
-#[pyclass(name = "TosRawClient")]
+#[pyclass(name = "TosRawClient", module = "tosnativeclient")]
 pub struct TosRawClient {
     client: Arc<InnerTosClient>,
     runtime: Arc<Runtime>,
@@ -158,7 +160,7 @@ impl TosRawClient {
                         content_range: output.content_range().to_string(),
                         hash_crc64ecma: output.hash_crc64ecma(),
                         output: Arc::new(Mutex::new(output)),
-                        runtime: runtime.clone(),
+                        runtime,
                     }),
                 }
             })
@@ -218,9 +220,24 @@ impl TosRawClient {
             })
         })
     }
+
+    pub fn __getnewargs__(slf: PyRef<'_, Self>) -> PyResult<Bound<'_, PyTuple>> {
+        let py = slf.py();
+        let state = [
+            slf.region.clone().into_pyobject(py)?.into_any(),
+            slf.endpoint.clone().into_pyobject(py)?.into_any(),
+            slf.ak.clone().into_pyobject(py)?.into_any(),
+            slf.sk.clone().into_pyobject(py)?.into_any(),
+            slf.connection_timeout.clone().into_pyobject(py)?.into_any(),
+            slf.request_timeout.clone().into_pyobject(py)?.into_any(),
+            slf.max_connections.clone().into_pyobject(py)?.into_any(),
+            slf.max_retry_count.clone().into_pyobject(py)?.into_any(),
+        ];
+        PyTuple::new(py, state)
+    }
 }
 
-#[pyclass(name = "HeadObjectInput")]
+#[pyclass(name = "HeadObjectInput", module = "tosnativeclient")]
 pub struct HeadObjectInput {
     #[pyo3(get, set)]
     pub(crate) bucket: String,
@@ -241,9 +258,18 @@ impl HeadObjectInput {
             version_id: version_id.to_string(),
         })
     }
+    pub fn __getnewargs__(slf: PyRef<'_, Self>) -> PyResult<Bound<'_, PyTuple>> {
+        let py = slf.py();
+        let state = [
+            slf.bucket.clone().into_pyobject(py)?.into_any(),
+            slf.key.clone().into_pyobject(py)?.into_any(),
+            slf.version_id.clone().into_pyobject(py)?.into_any(),
+        ];
+        PyTuple::new(py, state)
+    }
 }
 
-#[pyclass(name = "HeadObjectOutput")]
+#[pyclass(name = "HeadObjectOutput", module = "tosnativeclient")]
 pub struct HeadObjectOutput {
     #[pyo3(get)]
     pub(crate) request_id: String,
@@ -261,7 +287,7 @@ pub struct HeadObjectOutput {
     pub(crate) hash_crc64ecma: u64,
 }
 
-#[pyclass(name = "DeleteObjectInput")]
+#[pyclass(name = "DeleteObjectInput", module = "tosnativeclient")]
 pub struct DeleteObjectInput {
     #[pyo3(get, set)]
     pub(crate) bucket: String,
@@ -282,9 +308,18 @@ impl DeleteObjectInput {
             version_id: version_id.to_string(),
         })
     }
+    pub fn __getnewargs__(slf: PyRef<'_, Self>) -> PyResult<Bound<'_, PyTuple>> {
+        let py = slf.py();
+        let state = [
+            slf.bucket.clone().into_pyobject(py)?.into_any(),
+            slf.key.clone().into_pyobject(py)?.into_any(),
+            slf.version_id.clone().into_pyobject(py)?.into_any(),
+        ];
+        PyTuple::new(py, state)
+    }
 }
 
-#[pyclass(name = "DeleteObjectOutput")]
+#[pyclass(name = "DeleteObjectOutput", module = "tosnativeclient")]
 pub struct DeleteObjectOutput {
     #[pyo3(get)]
     pub(crate) request_id: String,
@@ -297,7 +332,7 @@ pub struct DeleteObjectOutput {
     #[pyo3(get)]
     pub(crate) version_id: String,
 }
-#[pyclass(name = "GetObjectInput")]
+#[pyclass(name = "GetObjectInput", module = "tosnativeclient")]
 pub struct GetObjectInput {
     #[pyo3(get, set)]
     pub(crate) bucket: String,
@@ -321,9 +356,19 @@ impl GetObjectInput {
             range: range.to_string(),
         })
     }
+    pub fn __getnewargs__(slf: PyRef<'_, Self>) -> PyResult<Bound<'_, PyTuple>> {
+        let py = slf.py();
+        let state = [
+            slf.bucket.clone().into_pyobject(py)?.into_any(),
+            slf.key.clone().into_pyobject(py)?.into_any(),
+            slf.version_id.clone().into_pyobject(py)?.into_any(),
+            slf.range.clone().into_pyobject(py)?.into_any(),
+        ];
+        PyTuple::new(py, state)
+    }
 }
 
-#[pyclass(name = "GetObjectOutput")]
+#[pyclass(name = "GetObjectOutput", module = "tosnativeclient")]
 pub struct GetObjectOutput {
     #[pyo3(get)]
     pub(crate) request_id: String,
@@ -350,30 +395,34 @@ impl GetObjectOutput {
     pub fn read_all(slf: PyRefMut<'_, Self>) -> PyResult<Option<Vec<u8>>> {
         let runtime = slf.runtime.clone();
         let output = slf.output.clone();
-        runtime.block_on(async move {
-            match output.lock().await.read_all().await {
-                Err(ex) => Err(map_tos_error(ex)),
-                Ok(buf) => Ok(Some(buf)),
-            }
+        slf.py().allow_threads(|| {
+            runtime.block_on(async move {
+                match output.lock().await.read_all().await {
+                    Err(ex) => Err(map_tos_error(ex)),
+                    Ok(buf) => Ok(Some(buf)),
+                }
+            })
         })
     }
 
     pub fn read(slf: PyRefMut<'_, Self>) -> PyResult<Option<Vec<u8>>> {
         let runtime = slf.runtime.clone();
         let output = slf.output.clone();
-        runtime.block_on(async move {
-            match output.lock().await.next().await {
-                None => Ok(None),
-                Some(result) => match result {
-                    Err(ex) => Err(map_error(ex)),
-                    Ok(buf) => Ok(Some(buf.chunk().to_vec())),
-                },
-            }
+        slf.py().allow_threads(|| {
+            runtime.block_on(async move {
+                match output.lock().await.next().await {
+                    None => Ok(None),
+                    Some(result) => match result {
+                        Err(ex) => Err(map_error(ex)),
+                        Ok(buf) => Ok(Some(buf.chunk().to_vec())),
+                    },
+                }
+            })
         })
     }
 }
 
-#[pyclass(name = "PutObjectFromBufferInput")]
+#[pyclass(name = "PutObjectFromBufferInput", module = "tosnativeclient")]
 pub struct PutObjectFromBufferInput {
     #[pyo3(get, set)]
     pub(crate) bucket: String,
@@ -394,9 +443,18 @@ impl PutObjectFromBufferInput {
             content: content.to_vec(),
         })
     }
+    pub fn __getnewargs__(slf: PyRef<'_, Self>) -> PyResult<Bound<'_, PyTuple>> {
+        let py = slf.py();
+        let state = [
+            slf.bucket.clone().into_pyobject(py)?.into_any(),
+            slf.key.clone().into_pyobject(py)?.into_any(),
+            slf.content.clone().into_pyobject(py)?.into_any(),
+        ];
+        PyTuple::new(py, state)
+    }
 }
 
-#[pyclass(name = "PutObjectFromFileInput")]
+#[pyclass(name = "PutObjectFromFileInput", module = "tosnativeclient")]
 pub struct PutObjectFromFileInput {
     #[pyo3(get, set)]
     pub(crate) bucket: String,
@@ -417,9 +475,18 @@ impl PutObjectFromFileInput {
             file_path: file_path.to_string(),
         })
     }
+    pub fn __getnewargs__(slf: PyRef<'_, Self>) -> PyResult<Bound<'_, PyTuple>> {
+        let py = slf.py();
+        let state = [
+            slf.bucket.clone().into_pyobject(py)?.into_any(),
+            slf.key.clone().into_pyobject(py)?.into_any(),
+            slf.file_path.clone().into_pyobject(py)?.into_any(),
+        ];
+        PyTuple::new(py, state)
+    }
 }
 
-#[pyclass(name = "PutObjectOutput")]
+#[pyclass(name = "PutObjectOutput", module = "tosnativeclient")]
 pub struct PutObjectOutput {
     #[pyo3(get)]
     pub(crate) request_id: String,

{tosnativeclient-1.0.1 → tosnativeclient-1.0.4}/src/write_stream.rs

@@ -2,7 +2,8 @@ use crate::tos_client::InnerTosClient;
 use crate::tos_error::map_tos_error;
 use async_channel::{Receiver, Sender};
 use futures_util::future::join_all;
-use pyo3::{pyclass, pymethods, PyRefMut, PyResult};
+use pyo3::types::PyTuple;
+use pyo3::{pyclass, pymethods, Bound, IntoPyObject, PyRef, PyRefMut, PyResult};
 use std::collections::HashMap;
 use std::sync::atomic::{AtomicI8, AtomicIsize, Ordering};
 use std::sync::Arc;
@@ -29,7 +30,7 @@ const MAX_PART_NUMBER: isize = 10000;
 const OTHER_MU_KICK_OFF: i8 = 1;
 const RELEASE_MU_KICK_OFF: i8 = 2;
 
-#[pyclass(name = "WriteStream")]
+#[pyclass(name = "WriteStream", module = "tosnativeclient")]
 pub struct WriteStream {
     object_writer: Arc<ObjectWriter>,
     runtime: Arc<Runtime>,
@@ -71,10 +72,20 @@ impl WriteStream {
             Ok(_) => Ok(()),
         }
     }
+
+    pub fn __getnewargs__(slf: PyRef<'_, Self>) -> PyResult<Bound<'_, PyTuple>> {
+        let py = slf.py();
+        let state = [
+            slf.bucket.clone().into_pyobject(py)?.into_any(),
+            slf.key.clone().into_pyobject(py)?.into_any(),
+            slf.storage_class.clone().into_pyobject(py)?.into_any(),
+        ];
+        PyTuple::new(py, state)
+    }
 }
 
 impl WriteStream {
-    pub(crate) fn new(
+    pub(crate) async fn new(
         client: Arc<InnerTosClient>,
         runtime: Arc<Runtime>,
         bucket: String,
@@ -93,10 +104,8 @@ impl WriteStream {
         let _key = key.clone();
         let _storage_class = storage_class.clone();
 
-        let object_writer = runtime.block_on(async move {
-            ObjectWriter::new(client, _bucket, _key, _storage_class, part_size).await
-        })?;
-
+        let object_writer =
+            ObjectWriter::new(client, _bucket, _key, _storage_class, part_size).await?;
         Ok(Self {
             object_writer: Arc::new(object_writer),
             runtime,

tosnativeclient-1.0.1/python/tosnativeclient/__init__.py

@@ -1,21 +0,0 @@
-from .tosnativeclient import TosClient, ListStream, ListObjectsResult, TosObject, TosError, TosException, TosRawClient, \
-    HeadObjectInput, HeadObjectOutput, GetObjectOutput, DeleteObjectInput, DeleteObjectOutput, GetObjectInput, \
-    PutObjectFromBufferInput, PutObjectFromFileInput, PutObjectOutput
-
-__all__ = ['TosClient',
-           'ListStream',
-           'ListObjectsResult',
-           'TosObject',
-           'TosError',
-           'TosException',
-           'TosRawClient',
-           'HeadObjectInput',
-           'HeadObjectOutput',
-           'DeleteObjectInput',
-           'DeleteObjectOutput',
-           'GetObjectInput',
-           'GetObjectOutput',
-           'PutObjectFromBufferInput',
-           'PutObjectFromFileInput',
-           'PutObjectOutput'
-           ]