tosnativeclient 1.0.1__tar.gz → 1.0.2__tar.gz

{tosnativeclient-1.0.1 → tosnativeclient-1.0.2}/Cargo.lock

@@ -472,6 +472,12 @@ version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
 
+[[package]]
+name = "hermit-abi"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c"
+
 [[package]]
 name = "hex"
 version = "0.4.3"
@@ -859,6 +865,16 @@ dependencies = [
  "autocfg",
 ]
 
+[[package]]
+name = "num_cpus"
+version = "1.17.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b"
+dependencies = [
+ "hermit-abi",
+ "libc",
+]
+
 [[package]]
 name = "object"
 version = "0.36.7"
@@ -1505,13 +1521,14 @@ dependencies = [
 
 [[package]]
 name = "tosnativeclient"
-version = "1.0.1"
+version = "1.0.2"
 dependencies = [
  "arc-swap",
  "async-channel",
  "async-trait",
  "bytes",
  "futures-util",
+ "num_cpus",
  "pyo3",
  "tokio",
  "tracing",

{tosnativeclient-1.0.1 → tosnativeclient-1.0.2}/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "tosnativeclient"
-version = "1.0.1"
+version = "1.0.2"
 edition = "2021"
 publish = false
 authors = ["xiangshijian"]
@@ -23,3 +23,4 @@ async-trait = "0.1.88"
 futures-util = "0.3.30"
 async-channel = "2.3.1"
 tracing = { version = "0.1.41", features = ["log"] }
+num_cpus = "1.17.0"

{tosnativeclient-1.0.1 → tosnativeclient-1.0.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tosnativeclient
-Version: 1.0.1
+Version: 1.0.2
 Classifier: Programming Language :: Rust
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy

tosnativeclient-1.0.2/python/tosnativeclient/__init__.py

@@ -0,0 +1,25 @@
+from .tosnativeclient import TosClient, ListStream, ListObjectsResult, TosObject, ReadStream, WriteStream, TosError, \
+    TosException, TosRawClient, \
+    HeadObjectInput, HeadObjectOutput, GetObjectOutput, DeleteObjectInput, DeleteObjectOutput, GetObjectInput, \
+    PutObjectFromBufferInput, PutObjectFromFileInput, PutObjectOutput
+
+__all__ = [
+    'TosError',
+    'TosException',
+    'TosClient',
+    'ListStream',
+    'ListObjectsResult',
+    'TosObject',
+    'ReadStream',
+    'WriteStream',
+    'TosRawClient',
+    'HeadObjectInput',
+    'HeadObjectOutput',
+    'DeleteObjectInput',
+    'DeleteObjectOutput',
+    'GetObjectInput',
+    'GetObjectOutput',
+    'PutObjectFromBufferInput',
+    'PutObjectFromFileInput',
+    'PutObjectOutput'
+]

{tosnativeclient-1.0.1 → tosnativeclient-1.0.2}/python/tosnativeclient/tosnativeclient.pyi

@@ -3,31 +3,27 @@ from typing import List, Dict
 from typing import Optional
 
 
-class TosClient(object):
-    region: str
-    endpoint: str
-    ak: str
-    sk: str
-    part_size: int
-    max_retry_count: int
-    max_prefetch_tasks: int
+class TosError(object):
+    message: str
+    status_code: Optional[int]
+    ec: str
+    request_id: str
 
-    def __init__(self, region: str, endpoint: str, ak: str = '', sk: str = '', part_size: int = 8388608,
-                 max_retry_count: int = 3, max_prefetch_tasks: int = 3, directives: str = '', directory: str = '',
-                 file_name_prefix: str = '', shared_prefetch_tasks: int = 20):
-        ...
 
-    def list_objects(self, bucket: str, prefix: str = '', max_keys: int = 1000, delimiter: str = '') -> ListStream:
-        ...
+class TosException(Exception):
+    args: List[TosError]
 
-    def head_object(self, bucket: str, key: str) -> TosObject:
-        ...
 
-    def get_object(self, bucket: str, key: str, etag: str, size: int) -> ReadStream:
-        ...
+class TosObject(object):
+    bucket: str
+    key: str
+    size: int
+    etag: str
 
-    def put_object(self, bucket: str, key: str, storage_class: str = '') -> WriteStream:
-        ...
+
+class ListObjectsResult(object):
+    contents: List[TosObject]
+    common_prefixes: List[str]
 
 
 class ListStream(object):
@@ -43,18 +39,6 @@ class ListStream(object):
     def close(self) -> None: ...
 
 
-class ListObjectsResult(object):
-    contents: List[TosObject]
-    common_prefixes: List[str]
-
-
-class TosObject(object):
-    bucket: str
-    key: str
-    size: int
-    etag: str
-
-
 class ReadStream(object):
     bucket: str
     key: str
@@ -80,44 +64,30 @@ class WriteStream(object):
         ...
 
 
-class TosError(object):
-    message: str
-    status_code: Optional[int]
-    ec: str
-    request_id: str
-
-
-class TosException(Exception):
-    args: List[TosError]
-
-
-class TosRawClient(object):
+class TosClient(object):
     region: str
     endpoint: str
     ak: str
     sk: str
-    connection_timeout: int
-    request_timeout: int
-    max_connections: int
+    part_size: int
     max_retry_count: int
+    max_prefetch_tasks: int
 
-    def __init__(self, region: str, endpoint: str, ak: str = '', sk: str = '', connection_timeout: int = 10000,
-                 request_timeout: int = 120000, max_connections: int = 1024, max_retry_count: int = 3):
-        ...
-
-    def head_object(self, input: HeadObjectInput) -> HeadObjectOutput:
+    def __init__(self, region: str, endpoint: str, ak: str = '', sk: str = '', part_size: int = 8388608,
+                 max_retry_count: int = 3, max_prefetch_tasks: int = 3, directives: str = '', directory: str = '',
+                 file_name_prefix: str = '', shared_prefetch_tasks: int = 20):
         ...
 
-    def delete_object(self, input: DeleteObjectInput) -> DeleteObjectOutput:
+    def list_objects(self, bucket: str, prefix: str = '', max_keys: int = 1000, delimiter: str = '') -> ListStream:
         ...
 
-    def get_object(self, input: GetObjectInput) -> GetObjectOutput:
+    def head_object(self, bucket: str, key: str) -> TosObject:
         ...
 
-    def put_object_from_buffer(self, input: PutObjectFromBufferInput) -> PutObjectOutput:
+    def get_object(self, bucket: str, key: str, etag: str, size: int) -> ReadStream:
         ...
 
-    def put_object_from_file(self, input: PutObjectFromFileInput) -> PutObjectOutput:
+    def put_object(self, bucket: str, key: str, storage_class: Optional[str] = '') -> WriteStream:
         ...
 
 
@@ -209,3 +179,33 @@ class PutObjectOutput(object):
     etag: str
     version_id: str
     hash_crc64ecma: int
+
+
+class TosRawClient(object):
+    region: str
+    endpoint: str
+    ak: str
+    sk: str
+    connection_timeout: int
+    request_timeout: int
+    max_connections: int
+    max_retry_count: int
+
+    def __init__(self, region: str, endpoint: str, ak: str = '', sk: str = '', connection_timeout: int = 10000,
+                 request_timeout: int = 120000, max_connections: int = 1024, max_retry_count: int = 3):
+        ...
+
+    def head_object(self, input: HeadObjectInput) -> HeadObjectOutput:
+        ...
+
+    def delete_object(self, input: DeleteObjectInput) -> DeleteObjectOutput:
+        ...
+
+    def get_object(self, input: GetObjectInput) -> GetObjectOutput:
+        ...
+
+    def put_object_from_buffer(self, input: PutObjectFromBufferInput) -> PutObjectOutput:
+        ...
+
+    def put_object_from_file(self, input: PutObjectFromFileInput) -> PutObjectOutput:
+        ...

{tosnativeclient-1.0.1 → tosnativeclient-1.0.2}/src/list_stream.rs

@@ -46,14 +46,11 @@ impl ListStream {
             return Err(map_error_from_string("ListStream is closed"));
         }
 
-        {
-            let mut pg = slf.paginator.write().unwrap();
-            if pg.is_none() {
-                *pg = slf.list_background(slf.py());
-            }
+        let mut pg = slf.paginator.write().unwrap();
+        if pg.is_none() {
+            *pg = slf.list_background(slf.py());
         }
-
-        slf.next_page(slf.paginator.read().unwrap().as_ref(), slf.py())
+        slf.next_page(pg.as_ref(), slf.py())
     }
 
     pub fn close(&self) {
@@ -100,67 +97,54 @@ impl ListStream {
         py.allow_threads(|| {
             self.runtime.spawn(async move {
                 let mut need_break = false;
-                if input.delimiter() == "/" {
-                    let mut prefixes = Vec::with_capacity(16);
-                    let mut last_page_end = false;
+                if input.delimiter() == "" {
                     loop {
-                        if last_page_end {
-                            if prefixes.is_empty() {
-                                let _ = sender
-                                    .send((
-                                        true,
-                                        Err(TosError::TosClientError {
-                                            message: "invalid status error".to_string(),
-                                            cause: None,
-                                        }),
-                                    ))
-                                    .await;
-                                break;
-                            }
-
-                            let prefix = prefixes.remove(0);
-                            input.set_prefix(prefix);
-                            input.set_start_after("");
-                            input.set_continuation_token("");
-                            last_page_end = false;
-                        }
                         let result = client.list_objects_type2(&input).await;
                         if let Ok(ref o) = result {
                             if o.is_truncated() {
                                 input.set_continuation_token(o.next_continuation_token());
                             } else {
-                                last_page_end = true;
-                            }
-
-                            for cp in o.common_prefixes() {
-                                prefixes.push(cp.prefix().to_string());
+                                need_break = true;
                             }
-                            need_break = last_page_end && prefixes.is_empty();
                         } else {
                             need_break = true;
                         }
-
                         if let Err(_) = sender.send((need_break, result)).await {
-                            need_break = true;
+                            need_break = true
                         }
                         if need_break {
                             break;
                         }
                     }
                 } else {
+                    let mut prefixes = Vec::with_capacity(16);
+                    let mut last_page_end = false;
                     loop {
+                        if last_page_end {
+                            let prefix = prefixes.remove(0);
+                            input.set_prefix(prefix);
+                            input.set_start_after("");
+                            input.set_continuation_token("");
+                            last_page_end = false;
+                        }
                         let result = client.list_objects_type2(&input).await;
                         if let Ok(ref o) = result {
                             if o.is_truncated() {
                                 input.set_continuation_token(o.next_continuation_token());
                             } else {
-                                need_break = true;
+                                last_page_end = true;
+                            }
+
+                            for cp in o.common_prefixes() {
+                                prefixes.push(cp.prefix().to_string());
                             }
+                            need_break = last_page_end && prefixes.is_empty();
                         } else {
                             need_break = true;
                         }
+
                         if let Err(_) = sender.send((need_break, result)).await {
-                            need_break = true
+                            need_break = true;
                         }
                         if need_break {
                             break;
@@ -169,7 +153,6 @@ impl ListStream {
                 }
             });
         });
-
         Some(Paginator {
             is_end: ArcSwap::new(Arc::new(false)),
             last_err: ArcSwap::new(Arc::new(None)),

{tosnativeclient-1.0.1 → tosnativeclient-1.0.2}/src/tos_client.rs

@@ -10,8 +10,7 @@ use std::future::Future;
 use std::sync::atomic::{AtomicIsize, Ordering};
 use std::sync::Arc;
 use std::time::Duration;
-use tokio::runtime;
-use tokio::runtime::{Handle, Runtime};
+use tokio::runtime::{Builder, Handle, Runtime};
 use tracing_appender::non_blocking::WorkerGuard;
 use ve_tos_rust_sdk::asynchronous::object::ObjectAPI;
 use ve_tos_rust_sdk::asynchronous::tos;
@@ -101,7 +100,12 @@ impl TosClient {
             _guard = Some(init_tracing_log(directives, directory, file_name_prefix));
         }
 
-        let runtime = Arc::new(runtime::Builder::new_multi_thread().enable_all().build()?);
+        let logical_cores = num_cpus::get();
+        let mut builder = Builder::new_multi_thread();
+        if logical_cores > 0 {
+            builder.worker_threads(logical_cores);
+        }
+        let runtime = Arc::new(builder.enable_all().build()?);
         let mut clients = Vec::with_capacity(2);
         for _ in 0..2 {
             match tos::builder()
@@ -188,22 +192,23 @@ impl TosClient {
 
     #[pyo3(signature = (bucket, key, storage_class=None))]
     pub fn put_object(
-        &self,
+        slf: PyRef<'_, Self>,
         bucket: String,
         key: String,
         storage_class: Option<String>,
     ) -> PyResult<WriteStream> {
-        match WriteStream::new(
-            self.wclient.clone(),
-            self.runtime.clone(),
-            bucket,
-            key,
-            storage_class,
-            self.part_size,
-        ) {
-            Err(ex) => Err(map_tos_error(ex)),
-            Ok(ws) => Ok(ws),
-        }
+        let client = slf.wclient.clone();
+        let runtime = slf.runtime.clone();
+        let part_size = slf.part_size;
+        slf.py().allow_threads(|| {
+            runtime.clone().block_on(async move {
+                match WriteStream::new(client, runtime, bucket, key, storage_class, part_size).await
+                {
+                    Err(ex) => Err(map_tos_error(ex)),
+                    Ok(ws) => Ok(ws),
+                }
+            })
+        })
     }
 }
 

{tosnativeclient-1.0.1 → tosnativeclient-1.0.2}/src/write_stream.rs

@@ -74,7 +74,7 @@ impl WriteStream {
 }
 
 impl WriteStream {
-    pub(crate) fn new(
+    pub(crate) async fn new(
         client: Arc<InnerTosClient>,
         runtime: Arc<Runtime>,
         bucket: String,
@@ -93,10 +93,8 @@ impl WriteStream {
         let _key = key.clone();
         let _storage_class = storage_class.clone();
 
-        let object_writer = runtime.block_on(async move {
-            ObjectWriter::new(client, _bucket, _key, _storage_class, part_size).await
-        })?;
-
+        let object_writer =
+            ObjectWriter::new(client, _bucket, _key, _storage_class, part_size).await?;
         Ok(Self {
             object_writer: Arc::new(object_writer),
             runtime,

tosnativeclient-1.0.1/python/tosnativeclient/__init__.py

@@ -1,21 +0,0 @@
-from .tosnativeclient import TosClient, ListStream, ListObjectsResult, TosObject, TosError, TosException, TosRawClient, \
-    HeadObjectInput, HeadObjectOutput, GetObjectOutput, DeleteObjectInput, DeleteObjectOutput, GetObjectInput, \
-    PutObjectFromBufferInput, PutObjectFromFileInput, PutObjectOutput
-
-__all__ = ['TosClient',
-           'ListStream',
-           'ListObjectsResult',
-           'TosObject',
-           'TosError',
-           'TosException',
-           'TosRawClient',
-           'HeadObjectInput',
-           'HeadObjectOutput',
-           'DeleteObjectInput',
-           'DeleteObjectOutput',
-           'GetObjectInput',
-           'GetObjectOutput',
-           'PutObjectFromBufferInput',
-           'PutObjectFromFileInput',
-           'PutObjectOutput'
-           ]