tosnativeclient 1.0.0__tar.gz → 1.0.2__tar.gz

{tosnativeclient-1.0.0 → tosnativeclient-1.0.2}/Cargo.lock

@@ -472,6 +472,12 @@ version = "0.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
 
+[[package]]
+name = "hermit-abi"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c"
+
 [[package]]
 name = "hex"
 version = "0.4.3"
@@ -859,6 +865,16 @@ dependencies = [
  "autocfg",
 ]
 
+[[package]]
+name = "num_cpus"
+version = "1.17.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b"
+dependencies = [
+ "hermit-abi",
+ "libc",
+]
+
 [[package]]
 name = "object"
 version = "0.36.7"
@@ -1505,13 +1521,14 @@ dependencies = [
 
 [[package]]
 name = "tosnativeclient"
-version = "1.0.0"
+version = "1.0.2"
 dependencies = [
  "arc-swap",
  "async-channel",
  "async-trait",
  "bytes",
  "futures-util",
+ "num_cpus",
  "pyo3",
  "tokio",
  "tracing",
@@ -1704,6 +1721,8 @@ dependencies = [
  "serde",
  "serde_json",
  "sha2",
+ "tokio",
+ "tokio-util",
  "tracing",
  "tracing-appender",
  "tracing-subscriber",

{tosnativeclient-1.0.0 → tosnativeclient-1.0.2}/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "tosnativeclient"
-version = "1.0.0"
+version = "1.0.2"
 edition = "2021"
 publish = false
 authors = ["xiangshijian"]
@@ -15,7 +15,7 @@ crate-type = ["cdylib"]
 [dependencies]
 arc-swap = "1.7.1"
 pyo3 = { version = "0.25.0", features = ["extension-module"] }
-ve-tos-rust-sdk = "2.3.0"
+ve-tos-rust-sdk = { version = "2.3.0", features = ["async-file"] }
 tracing-appender = "0.2.3"
 bytes = "1.10.1"
 tokio = { version = "1.36.0", features = ["rt-multi-thread"] }
@@ -23,3 +23,4 @@ async-trait = "0.1.88"
 futures-util = "0.3.30"
 async-channel = "2.3.1"
 tracing = { version = "0.1.41", features = ["log"] }
+num_cpus = "1.17.0"

{tosnativeclient-1.0.0 → tosnativeclient-1.0.2}/PKG-INFO

@@ -1,9 +1,10 @@
 Metadata-Version: 2.4
 Name: tosnativeclient
-Version: 1.0.0
+Version: 1.0.2
 Classifier: Programming Language :: Rust
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy
+Classifier: License :: OSI Approved :: Apache Software License
 Summary: python native client for tos
 Author: xiangshijian
 License: Apache-2.0

{tosnativeclient-1.0.0 → tosnativeclient-1.0.2}/pyproject.toml

@@ -10,6 +10,7 @@ classifiers = [
     "Programming Language :: Rust",
     "Programming Language :: Python :: Implementation :: CPython",
     "Programming Language :: Python :: Implementation :: PyPy",
+    "License :: OSI Approved :: Apache Software License",
 ]
 dynamic = ["version"]
 [tool.maturin]

tosnativeclient-1.0.2/python/tosnativeclient/__init__.py

@@ -0,0 +1,25 @@
+from .tosnativeclient import TosClient, ListStream, ListObjectsResult, TosObject, ReadStream, WriteStream, TosError, \
+    TosException, TosRawClient, \
+    HeadObjectInput, HeadObjectOutput, GetObjectOutput, DeleteObjectInput, DeleteObjectOutput, GetObjectInput, \
+    PutObjectFromBufferInput, PutObjectFromFileInput, PutObjectOutput
+
+__all__ = [
+    'TosError',
+    'TosException',
+    'TosClient',
+    'ListStream',
+    'ListObjectsResult',
+    'TosObject',
+    'ReadStream',
+    'WriteStream',
+    'TosRawClient',
+    'HeadObjectInput',
+    'HeadObjectOutput',
+    'DeleteObjectInput',
+    'DeleteObjectOutput',
+    'GetObjectInput',
+    'GetObjectOutput',
+    'PutObjectFromBufferInput',
+    'PutObjectFromFileInput',
+    'PutObjectOutput'
+]

tosnativeclient-1.0.2/python/tosnativeclient/tosnativeclient.pyi

@@ -0,0 +1,211 @@
+from typing import List, Dict
+
+from typing import Optional
+
+
+class TosError(object):
+    message: str
+    status_code: Optional[int]
+    ec: str
+    request_id: str
+
+
+class TosException(Exception):
+    args: List[TosError]
+
+
+class TosObject(object):
+    bucket: str
+    key: str
+    size: int
+    etag: str
+
+
+class ListObjectsResult(object):
+    contents: List[TosObject]
+    common_prefixes: List[str]
+
+
+class ListStream(object):
+    bucket: str
+    prefix: str
+    delimiter: str
+    max_keys: int
+
+    def __iter__(self) -> ListStream: ...
+
+    def __next__(self) -> ListObjectsResult: ...
+
+    def close(self) -> None: ...
+
+
+class ReadStream(object):
+    bucket: str
+    key: str
+    size: int
+    etag: str
+
+    def read(self, offset: int, length: int) -> bytes:
+        ...
+
+    def close(self) -> None:
+        ...
+
+
+class WriteStream(object):
+    bucket: str
+    key: str
+    storage_class: Optional[str]
+
+    def write(self, data: bytes) -> int:
+        ...
+
+    def close(self) -> None:
+        ...
+
+
+class TosClient(object):
+    region: str
+    endpoint: str
+    ak: str
+    sk: str
+    part_size: int
+    max_retry_count: int
+    max_prefetch_tasks: int
+
+    def __init__(self, region: str, endpoint: str, ak: str = '', sk: str = '', part_size: int = 8388608,
+                 max_retry_count: int = 3, max_prefetch_tasks: int = 3, directives: str = '', directory: str = '',
+                 file_name_prefix: str = '', shared_prefetch_tasks: int = 20):
+        ...
+
+    def list_objects(self, bucket: str, prefix: str = '', max_keys: int = 1000, delimiter: str = '') -> ListStream:
+        ...
+
+    def head_object(self, bucket: str, key: str) -> TosObject:
+        ...
+
+    def get_object(self, bucket: str, key: str, etag: str, size: int) -> ReadStream:
+        ...
+
+    def put_object(self, bucket: str, key: str, storage_class: Optional[str] = '') -> WriteStream:
+        ...
+
+
+class HeadObjectInput(object):
+    bucket: str
+    key: str
+    version_id: str
+
+    def __init__(self, bucket: str, key: str, version_id: str = ''):
+        ...
+
+
+class HeadObjectOutput(object):
+    request_id: str
+    status_code: int
+    header: Dict[str, str]
+    content_length: int
+    etag: str
+    version_id: str
+    hash_crc64ecma: int
+
+
+class DeleteObjectInput(object):
+    bucket: str
+    key: str
+    version_id: str
+
+    def __init__(self, bucket: str, key: str, version_id: str = ''):
+        ...
+
+
+class DeleteObjectOutput(object):
+    request_id: str
+    status_code: int
+    header: Dict[str, str]
+    delete_marker: bool
+    version_id: str
+
+
+class GetObjectInput(object):
+    bucket: str
+    key: str
+    version_id: str
+    range: str
+
+    def __init__(self, bucket: str, key: str, version_id: str = '', range: str = ''):
+        ...
+
+
+class GetObjectOutput(object):
+    request_id: str
+    status_code: int
+    header: Dict[str, str]
+    content_length: int
+    etag: str
+    version_id: str
+    content_range: str
+    hash_crc64ecma: int
+
+    def read_all(self) -> bytes:
+        ...
+
+    def read(self) -> bytes:
+        ...
+
+
+class PutObjectFromBufferInput(object):
+    bucket: str
+    key: str
+    content: bytes
+
+    def __init__(self, bucket: str, key: str, content: bytes):
+        ...
+
+
+class PutObjectFromFileInput(object):
+    bucket: str
+    key: str
+    file_path: str
+
+    def __init__(self, bucket: str, key: str, file_path: str):
+        ...
+
+
+class PutObjectOutput(object):
+    request_id: str
+    status_code: int
+    header: Dict[str, str]
+    etag: str
+    version_id: str
+    hash_crc64ecma: int
+
+
+class TosRawClient(object):
+    region: str
+    endpoint: str
+    ak: str
+    sk: str
+    connection_timeout: int
+    request_timeout: int
+    max_connections: int
+    max_retry_count: int
+
+    def __init__(self, region: str, endpoint: str, ak: str = '', sk: str = '', connection_timeout: int = 10000,
+                 request_timeout: int = 120000, max_connections: int = 1024, max_retry_count: int = 3):
+        ...
+
+    def head_object(self, input: HeadObjectInput) -> HeadObjectOutput:
+        ...
+
+    def delete_object(self, input: DeleteObjectInput) -> DeleteObjectOutput:
+        ...
+
+    def get_object(self, input: GetObjectInput) -> GetObjectOutput:
+        ...
+
+    def put_object_from_buffer(self, input: PutObjectFromBufferInput) -> PutObjectOutput:
+        ...
+
+    def put_object_from_file(self, input: PutObjectFromFileInput) -> PutObjectOutput:
+        ...

{tosnativeclient-1.0.0 → tosnativeclient-1.0.2}/src/lib.rs

@@ -3,6 +3,11 @@ use crate::read_stream::ReadStream;
 use crate::tos_client::TosClient;
 use crate::tos_error::{TosError, TosException};
 use crate::tos_model::{ListObjectsResult, TosObject};
+use crate::tos_raw_client::{
+    DeleteObjectInput, DeleteObjectOutput, GetObjectInput, GetObjectOutput, HeadObjectInput,
+    HeadObjectOutput, PutObjectFromBufferInput, PutObjectFromFileInput, PutObjectOutput,
+    TosRawClient,
+};
 use crate::write_stream::WriteStream;
 use pyo3::prelude::*;
 
@@ -11,11 +16,22 @@ mod read_stream;
 mod tos_client;
 mod tos_error;
 mod tos_model;
+mod tos_raw_client;
 mod write_stream;
 
 #[pymodule]
 #[pyo3(name = "tosnativeclient")]
 fn main(m: &Bound<'_, PyModule>) -> PyResult<()> {
+    m.add_class::<TosRawClient>()?;
+    m.add_class::<HeadObjectInput>()?;
+    m.add_class::<HeadObjectOutput>()?;
+    m.add_class::<DeleteObjectInput>()?;
+    m.add_class::<DeleteObjectOutput>()?;
+    m.add_class::<GetObjectInput>()?;
+    m.add_class::<GetObjectOutput>()?;
+    m.add_class::<PutObjectFromBufferInput>()?;
+    m.add_class::<PutObjectFromFileInput>()?;
+    m.add_class::<PutObjectOutput>()?;
     m.add_class::<TosClient>()?;
     m.add_class::<ListStream>()?;
     m.add_class::<ListObjectsResult>()?;

{tosnativeclient-1.0.0 → tosnativeclient-1.0.2}/src/list_stream.rs

@@ -1,5 +1,5 @@
 use crate::tos_client::InnerTosClient;
-use crate::tos_error::{map_error, map_error_from_string};
+use crate::tos_error::{map_error_from_string, map_tos_error};
 use crate::tos_model::ListObjectsResult;
 use arc_swap::ArcSwap;
 use async_channel::Receiver;
@@ -46,14 +46,11 @@ impl ListStream {
             return Err(map_error_from_string("ListStream is closed"));
         }
 
-        {
-            let mut pg = slf.paginator.write().unwrap();
-            if pg.is_none() {
-                *pg = slf.list_background(slf.py());
-            }
+        let mut pg = slf.paginator.write().unwrap();
+        if pg.is_none() {
+            *pg = slf.list_background(slf.py());
         }
-
-        slf.next_page(slf.paginator.read().unwrap().as_ref(), slf.py())
+        slf.next_page(pg.as_ref(), slf.py())
     }
 
     pub fn close(&self) {
@@ -100,67 +97,54 @@ impl ListStream {
         py.allow_threads(|| {
             self.runtime.spawn(async move {
                 let mut need_break = false;
-                if input.delimiter() == "/" {
-                    let mut prefixes = Vec::with_capacity(16);
-                    let mut last_page_end = false;
+                if input.delimiter() == "" {
                     loop {
-                        if last_page_end {
-                            if prefixes.is_empty() {
-                                let _ = sender
-                                    .send((
-                                        true,
-                                        Err(TosError::TosClientError {
-                                            message: "invalid status error".to_string(),
-                                            cause: None,
-                                        }),
-                                    ))
-                                    .await;
-                                break;
-                            }
-
-                            let prefix = prefixes.remove(0);
-                            input.set_prefix(prefix);
-                            input.set_start_after("");
-                            input.set_continuation_token("");
-                            last_page_end = false;
-                        }
                         let result = client.list_objects_type2(&input).await;
                         if let Ok(ref o) = result {
                             if o.is_truncated() {
                                 input.set_continuation_token(o.next_continuation_token());
                             } else {
-                                last_page_end = true;
-                            }
-
-                            for cp in o.common_prefixes() {
-                                prefixes.push(cp.prefix().to_string());
+                                need_break = true;
                             }
-                            need_break = last_page_end && prefixes.is_empty();
                         } else {
                             need_break = true;
                         }
-
                         if let Err(_) = sender.send((need_break, result)).await {
-                            need_break = true;
+                            need_break = true
                         }
                         if need_break {
                             break;
                         }
                     }
                 } else {
+                    let mut prefixes = Vec::with_capacity(16);
+                    let mut last_page_end = false;
                     loop {
+                        if last_page_end {
+                            let prefix = prefixes.remove(0);
+                            input.set_prefix(prefix);
+                            input.set_start_after("");
+                            input.set_continuation_token("");
+                            last_page_end = false;
+                        }
                         let result = client.list_objects_type2(&input).await;
                         if let Ok(ref o) = result {
                             if o.is_truncated() {
                                 input.set_continuation_token(o.next_continuation_token());
                             } else {
-                                need_break = true;
+                                last_page_end = true;
+                            }
+
+                            for cp in o.common_prefixes() {
+                                prefixes.push(cp.prefix().to_string());
                             }
+                            need_break = last_page_end && prefixes.is_empty();
                         } else {
                             need_break = true;
                         }
+
                         if let Err(_) = sender.send((need_break, result)).await {
-                            need_break = true
+                            need_break = true;
                         }
                         if need_break {
                             break;
@@ -169,7 +153,6 @@ impl ListStream {
                 }
             });
         });
-
         Some(Paginator {
             is_end: ArcSwap::new(Arc::new(false)),
             last_err: ArcSwap::new(Arc::new(None)),
@@ -187,7 +170,7 @@ impl ListStream {
             None => Ok(None),
             Some(pg) => {
                 match pg.has_next() {
-                    Err(ex) => return Err(map_error(ex)),
+                    Err(ex) => return Err(map_tos_error(ex)),
                     Ok(has_next) => {
                         if !has_next {
                             return Ok(None);
@@ -199,7 +182,7 @@ impl ListStream {
                     self.runtime.block_on(async {
                         match pg.next_page().await {
                             Ok(output) => Ok(Some(ListObjectsResult::new(output))),
-                            Err(ex) => Err(map_error(ex)),
+                            Err(ex) => Err(map_tos_error(ex)),
                         }
                     })
                 })

{tosnativeclient-1.0.0 → tosnativeclient-1.0.2}/src/read_stream.rs

@@ -1,5 +1,5 @@
 use crate::tos_client::{InnerTosClient, SharedPrefetchContext};
-use crate::tos_error::map_error;
+use crate::tos_error::map_tos_error;
 use crate::tos_model::TosObject;
 use async_channel::{Receiver, Sender};
 use bytes::Bytes;
@@ -52,7 +52,7 @@ impl ReadStream {
             .py()
             .allow_threads(|| runtime.block_on(async move { fetcher.read(offset, length).await }))
         {
-            Err(ex) => Err(map_error(ex)),
+            Err(ex) => Err(map_tos_error(ex)),
             Ok(result) => match result {
                 None => Ok(None),
                 Some(data) => Ok(Some(PyBytes::new(slf.py(), data.as_ref()))),

{tosnativeclient-1.0.0 → tosnativeclient-1.0.2}/src/tos_client.rs

@@ -1,6 +1,6 @@
 use crate::list_stream::ListStream;
 use crate::read_stream::ReadStream;
-use crate::tos_error::map_error;
+use crate::tos_error::map_tos_error;
 use crate::tos_model::TosObject;
 use crate::write_stream::WriteStream;
 use async_trait::async_trait;
@@ -10,8 +10,7 @@ use std::future::Future;
 use std::sync::atomic::{AtomicIsize, Ordering};
 use std::sync::Arc;
 use std::time::Duration;
-use tokio::runtime;
-use tokio::runtime::{Handle, Runtime};
+use tokio::runtime::{Builder, Handle, Runtime};
 use tracing_appender::non_blocking::WorkerGuard;
 use ve_tos_rust_sdk::asynchronous::object::ObjectAPI;
 use ve_tos_rust_sdk::asynchronous::tos;
@@ -101,7 +100,12 @@ impl TosClient {
             _guard = Some(init_tracing_log(directives, directory, file_name_prefix));
         }
 
-        let runtime = Arc::new(runtime::Builder::new_multi_thread().enable_all().build()?);
+        let logical_cores = num_cpus::get();
+        let mut builder = Builder::new_multi_thread();
+        if logical_cores > 0 {
+            builder.worker_threads(logical_cores);
+        }
+        let runtime = Arc::new(builder.enable_all().build()?);
         let mut clients = Vec::with_capacity(2);
         for _ in 0..2 {
             match tos::builder()
@@ -118,7 +122,7 @@ impl TosClient {
                 })
                 .build()
             {
-                Err(ex) => return Err(map_error(ex)),
+                Err(ex) => return Err(map_tos_error(ex)),
                 Ok(client) => {
                     clients.push(client);
                 }
@@ -165,7 +169,7 @@ impl TosClient {
         slf.py().allow_threads(|| {
             runtime.block_on(async move {
                 match client.head_object(&input).await {
-                    Err(ex) => Err(map_error(ex)),
+                    Err(ex) => Err(map_tos_error(ex)),
                     Ok(output) => Ok(TosObject::new(input.bucket(), input.key(), output)),
                 }
             })
@@ -188,22 +192,23 @@ impl TosClient {
 
     #[pyo3(signature = (bucket, key, storage_class=None))]
     pub fn put_object(
-        &self,
+        slf: PyRef<'_, Self>,
         bucket: String,
         key: String,
         storage_class: Option<String>,
     ) -> PyResult<WriteStream> {
-        match WriteStream::new(
-            self.wclient.clone(),
-            self.runtime.clone(),
-            bucket,
-            key,
-            storage_class,
-            self.part_size,
-        ) {
-            Err(ex) => Err(map_error(ex)),
-            Ok(ws) => Ok(ws),
-        }
+        let client = slf.wclient.clone();
+        let runtime = slf.runtime.clone();
+        let part_size = slf.part_size;
+        slf.py().allow_threads(|| {
+            runtime.clone().block_on(async move {
+                match WriteStream::new(client, runtime, bucket, key, storage_class, part_size).await
+                {
+                    Err(ex) => Err(map_tos_error(ex)),
+                    Ok(ws) => Ok(ws),
+                }
+            })
+        })
     }
 }
 

{tosnativeclient-1.0.0 → tosnativeclient-1.0.2}/src/tos_error.rs

@@ -1,5 +1,6 @@
 use pyo3::exceptions::PyException;
 use pyo3::{create_exception, pyclass, pymethods, PyErr};
+use std::error::Error;
 
 #[pyclass]
 #[derive(Clone)]
@@ -55,7 +56,11 @@ pub(crate) fn map_error_from_string(message: impl Into<String>) -> PyErr {
     ))
 }
 
-pub(crate) fn map_error(err: ve_tos_rust_sdk::error::TosError) -> PyErr {
+pub(crate) fn map_error(err: impl Error) -> PyErr {
+    PyErr::new::<TosException, _>(TosError::message(err.to_string()))
+}
+
+pub(crate) fn map_tos_error(err: ve_tos_rust_sdk::error::TosError) -> PyErr {
     match err {
         ve_tos_rust_sdk::error::TosError::TosClientError { message, .. } => {
             PyErr::new::<TosException, _>(TosError::message(message))

tosnativeclient-1.0.2/src/tos_raw_client.rs

@@ -0,0 +1,436 @@
+use crate::tos_client::{InnerTosClient, TokioRuntime};
+use crate::tos_error::{map_error, map_tos_error};
+use bytes::Buf;
+use futures_util::StreamExt;
+use pyo3::{pyclass, pymethods, PyRef, PyRefMut, PyResult};
+use std::collections::HashMap;
+use std::sync::Arc;
+use tokio::runtime;
+use tokio::runtime::Runtime;
+use tokio::sync::Mutex;
+use ve_tos_rust_sdk::asynchronous::object::{ObjectAPI, ObjectContent};
+use ve_tos_rust_sdk::asynchronous::tos;
+
+#[pyclass(name = "TosRawClient")]
+pub struct TosRawClient {
+    client: Arc<InnerTosClient>,
+    runtime: Arc<Runtime>,
+    #[pyo3(get)]
+    region: String,
+    #[pyo3(get)]
+    endpoint: String,
+    #[pyo3(get)]
+    ak: String,
+    #[pyo3(get)]
+    sk: String,
+    #[pyo3(get)]
+    connection_timeout: isize,
+    #[pyo3(get)]
+    request_timeout: isize,
+    #[pyo3(get)]
+    max_connections: isize,
+    #[pyo3(get)]
+    max_retry_count: isize,
+}
+
+#[pymethods]
+impl TosRawClient {
+    #[new]
+    #[pyo3(signature = (region, endpoint, ak="", sk="", connection_timeout=10000, request_timeout=120000, max_connections=1024, max_retry_count=3)
+    )]
+    pub fn new(
+        region: &str,
+        endpoint: &str,
+        ak: &str,
+        sk: &str,
+        connection_timeout: isize,
+        request_timeout: isize,
+        max_connections: isize,
+        max_retry_count: isize,
+    ) -> PyResult<Self> {
+        let runtime = Arc::new(runtime::Builder::new_multi_thread().enable_all().build()?);
+        match tos::builder()
+            .connection_timeout(connection_timeout)
+            .request_timeout(request_timeout)
+            .max_connections(max_connections)
+            .max_retry_count(max_retry_count)
+            .ak(ak)
+            .sk(sk)
+            .region(region)
+            .endpoint(endpoint)
+            .async_sleeper(TokioRuntime {
+                runtime: Some(runtime.clone()),
+            })
+            .build()
+        {
+            Err(ex) => Err(map_tos_error(ex)),
+            Ok(client) => Ok(Self {
+                client: Arc::new(client),
+                runtime,
+                region: region.to_string(),
+                endpoint: endpoint.to_string(),
+                ak: ak.to_string(),
+                sk: sk.to_string(),
+                connection_timeout,
+                request_timeout,
+                max_connections,
+                max_retry_count,
+            }),
+        }
+    }
+    pub fn head_object<'py>(
+        slf: PyRef<'_, Self>,
+        input: &HeadObjectInput,
+    ) -> PyResult<HeadObjectOutput> {
+        let input = ve_tos_rust_sdk::object::HeadObjectInput::new_with_version_id(
+            &input.bucket,
+            &input.key,
+            &input.version_id,
+        );
+        let client = slf.client.clone();
+        let runtime = slf.runtime.clone();
+        slf.py().allow_threads(|| {
+            runtime.block_on(async move {
+                match client.head_object(&input).await {
+                    Err(ex) => Err(map_tos_error(ex)),
+                    Ok(output) => Ok(HeadObjectOutput {
+                        request_id: output.request_id().to_string(),
+                        status_code: output.status_code(),
+                        header: output.header().clone(),
+                        content_length: output.content_length(),
+                        etag: output.etag().to_string(),
+                        version_id: output.version_id().to_string(),
+                        hash_crc64ecma: output.hash_crc64ecma(),
+                    }),
+                }
+            })
+        })
+    }
+
+    pub fn delete_object(
+        slf: PyRef<'_, Self>,
+        input: &DeleteObjectInput,
+    ) -> PyResult<DeleteObjectOutput> {
+        let input = ve_tos_rust_sdk::object::DeleteObjectInput::new_with_version_id(
+            &input.bucket,
+            &input.key,
+            &input.version_id,
+        );
+        let client = slf.client.clone();
+        let runtime = slf.runtime.clone();
+        slf.py().allow_threads(|| {
+            runtime.block_on(async move {
+                match client.delete_object(&input).await {
+                    Err(ex) => Err(map_tos_error(ex)),
+                    Ok(output) => Ok(DeleteObjectOutput {
+                        request_id: output.request_id().to_string(),
+                        status_code: output.status_code(),
+                        header: output.header().clone(),
+                        delete_marker: output.delete_marker(),
+                        version_id: output.version_id().to_string(),
+                    }),
+                }
+            })
+        })
+    }
+
+    pub fn get_object(slf: PyRef<'_, Self>, input: &GetObjectInput) -> PyResult<GetObjectOutput> {
+        let mut rinput = ve_tos_rust_sdk::object::GetObjectInput::new_with_version_id(
+            &input.bucket,
+            &input.key,
+            &input.version_id,
+        );
+        rinput.set_range(&input.range);
+
+        let client = slf.client.clone();
+        let runtime = slf.runtime.clone();
+        slf.py().allow_threads(|| {
+            runtime.clone().block_on(async move {
+                match client.get_object(&rinput).await {
+                    Err(ex) => Err(map_tos_error(ex)),
+                    Ok(output) => Ok(GetObjectOutput {
+                        request_id: output.request_id().to_string(),
+                        status_code: output.status_code(),
+                        header: output.header().clone(),
+                        content_length: output.content_length(),
+                        etag: output.etag().to_string(),
+                        version_id: output.version_id().to_string(),
+                        content_range: output.content_range().to_string(),
+                        hash_crc64ecma: output.hash_crc64ecma(),
+                        output: Arc::new(Mutex::new(output)),
+                        runtime: runtime.clone(),
+                    }),
+                }
+            })
+        })
+    }
+
+    pub fn put_object_from_buffer(
+        slf: PyRef<'_, Self>,
+        input: &PutObjectFromBufferInput,
+    ) -> PyResult<PutObjectOutput> {
+        let mut rinput =
+            ve_tos_rust_sdk::object::PutObjectFromBufferInput::new(&input.bucket, &input.key);
+        if input.content.len() > 0 {
+            rinput.set_content(input.content.as_slice());
+        }
+        let client = slf.client.clone();
+        let runtime = slf.runtime.clone();
+        slf.py().allow_threads(|| {
+            runtime.clone().block_on(async move {
+                match client.put_object_from_buffer(&rinput).await {
+                    Err(ex) => Err(map_tos_error(ex)),
+                    Ok(output) => Ok(PutObjectOutput {
+                        request_id: output.request_id().to_string(),
+                        status_code: output.status_code(),
+                        header: output.header().clone(),
+                        etag: output.etag().to_string(),
+                        version_id: output.version_id().to_string(),
+                        hash_crc64ecma: output.hash_crc64ecma(),
+                    }),
+                }
+            })
+        })
+    }
+
+    pub fn put_object_from_file(
+        slf: PyRef<'_, Self>,
+        input: &PutObjectFromFileInput,
+    ) -> PyResult<PutObjectOutput> {
+        let mut rinput =
+            ve_tos_rust_sdk::object::PutObjectFromFileInput::new(&input.bucket, &input.key);
+        rinput.set_file_path(&input.file_path);
+        let client = slf.client.clone();
+        let runtime = slf.runtime.clone();
+        slf.py().allow_threads(|| {
+            runtime.clone().block_on(async move {
+                match client.put_object_from_file(&rinput).await {
+                    Err(ex) => Err(map_tos_error(ex)),
+                    Ok(output) => Ok(PutObjectOutput {
+                        request_id: output.request_id().to_string(),
+                        status_code: output.status_code(),
+                        header: output.header().clone(),
+                        etag: output.etag().to_string(),
+                        version_id: output.version_id().to_string(),
+                        hash_crc64ecma: output.hash_crc64ecma(),
+                    }),
+                }
+            })
+        })
+    }
+}
+
+#[pyclass(name = "HeadObjectInput")]
+pub struct HeadObjectInput {
+    #[pyo3(get, set)]
+    pub(crate) bucket: String,
+    #[pyo3(get, set)]
+    pub(crate) key: String,
+    #[pyo3(get, set)]
+    pub(crate) version_id: String,
+}
+
+#[pymethods]
+impl HeadObjectInput {
+    #[new]
+    #[pyo3(signature = (bucket, key, version_id = ""))]
+    pub fn new(bucket: &str, key: &str, version_id: &str) -> PyResult<Self> {
+        Ok(Self {
+            bucket: bucket.to_string(),
+            key: key.to_string(),
+            version_id: version_id.to_string(),
+        })
+    }
+}
+
+#[pyclass(name = "HeadObjectOutput")]
+pub struct HeadObjectOutput {
+    #[pyo3(get)]
+    pub(crate) request_id: String,
+    #[pyo3(get)]
+    pub(crate) status_code: isize,
+    #[pyo3(get)]
+    pub(crate) header: HashMap<String, String>,
+    #[pyo3(get)]
+    pub(crate) content_length: i64,
+    #[pyo3(get)]
+    pub(crate) etag: String,
+    #[pyo3(get)]
+    pub(crate) version_id: String,
+    #[pyo3(get)]
+    pub(crate) hash_crc64ecma: u64,
+}
+
+#[pyclass(name = "DeleteObjectInput")]
+pub struct DeleteObjectInput {
+    #[pyo3(get, set)]
+    pub(crate) bucket: String,
+    #[pyo3(get, set)]
+    pub(crate) key: String,
+    #[pyo3(get, set)]
+    pub(crate) version_id: String,
+}
+
+#[pymethods]
+impl DeleteObjectInput {
+    #[new]
+    #[pyo3(signature = (bucket, key, version_id = ""))]
+    pub fn new(bucket: &str, key: &str, version_id: &str) -> PyResult<Self> {
+        Ok(Self {
+            bucket: bucket.to_string(),
+            key: key.to_string(),
+            version_id: version_id.to_string(),
+        })
+    }
+}
+
+#[pyclass(name = "DeleteObjectOutput")]
+pub struct DeleteObjectOutput {
+    #[pyo3(get)]
+    pub(crate) request_id: String,
+    #[pyo3(get)]
+    pub(crate) status_code: isize,
+    #[pyo3(get)]
+    pub(crate) header: HashMap<String, String>,
+    #[pyo3(get)]
+    pub(crate) delete_marker: bool,
+    #[pyo3(get)]
+    pub(crate) version_id: String,
+}
+#[pyclass(name = "GetObjectInput")]
+pub struct GetObjectInput {
+    #[pyo3(get, set)]
+    pub(crate) bucket: String,
+    #[pyo3(get, set)]
+    pub(crate) key: String,
+    #[pyo3(get, set)]
+    pub(crate) version_id: String,
+    #[pyo3(get, set)]
+    pub(crate) range: String,
+}
+
+#[pymethods]
+impl GetObjectInput {
+    #[new]
+    #[pyo3(signature = (bucket, key, version_id = "", range = ""))]
+    pub fn new(bucket: &str, key: &str, version_id: &str, range: &str) -> PyResult<Self> {
+        Ok(Self {
+            bucket: bucket.to_string(),
+            key: key.to_string(),
+            version_id: version_id.to_string(),
+            range: range.to_string(),
+        })
+    }
+}
+
+#[pyclass(name = "GetObjectOutput")]
+pub struct GetObjectOutput {
+    #[pyo3(get)]
+    pub(crate) request_id: String,
+    #[pyo3(get)]
+    pub(crate) status_code: isize,
+    #[pyo3(get)]
+    pub(crate) header: HashMap<String, String>,
+    #[pyo3(get)]
+    pub(crate) content_length: i64,
+    #[pyo3(get)]
+    pub(crate) etag: String,
+    #[pyo3(get)]
+    pub(crate) version_id: String,
+    #[pyo3(get)]
+    pub(crate) content_range: String,
+    #[pyo3(get)]
+    pub(crate) hash_crc64ecma: u64,
+    output: Arc<Mutex<ve_tos_rust_sdk::object::GetObjectOutput>>,
+    runtime: Arc<Runtime>,
+}
+
+#[pymethods]
+impl GetObjectOutput {
+    pub fn read_all(slf: PyRefMut<'_, Self>) -> PyResult<Option<Vec<u8>>> {
+        let runtime = slf.runtime.clone();
+        let output = slf.output.clone();
+        runtime.block_on(async move {
+            match output.lock().await.read_all().await {
+                Err(ex) => Err(map_tos_error(ex)),
+                Ok(buf) => Ok(Some(buf)),
+            }
+        })
+    }
+
+    pub fn read(slf: PyRefMut<'_, Self>) -> PyResult<Option<Vec<u8>>> {
+        let runtime = slf.runtime.clone();
+        let output = slf.output.clone();
+        runtime.block_on(async move {
+            match output.lock().await.next().await {
+                None => Ok(None),
+                Some(result) => match result {
+                    Err(ex) => Err(map_error(ex)),
+                    Ok(buf) => Ok(Some(buf.chunk().to_vec())),
+                },
+            }
+        })
+    }
+}
+
+#[pyclass(name = "PutObjectFromBufferInput")]
+pub struct PutObjectFromBufferInput {
+    #[pyo3(get, set)]
+    pub(crate) bucket: String,
+    #[pyo3(get, set)]
+    pub(crate) key: String,
+    #[pyo3(get, set)]
+    pub(crate) content: Vec<u8>,
+}
+
+#[pymethods]
+impl PutObjectFromBufferInput {
+    #[new]
+    #[pyo3(signature = (bucket, key, content))]
+    pub fn new(bucket: &str, key: &str, content: &[u8]) -> PyResult<Self> {
+        Ok(Self {
+            bucket: bucket.to_string(),
+            key: key.to_string(),
+            content: content.to_vec(),
+        })
+    }
+}
+
+#[pyclass(name = "PutObjectFromFileInput")]
+pub struct PutObjectFromFileInput {
+    #[pyo3(get, set)]
+    pub(crate) bucket: String,
+    #[pyo3(get, set)]
+    pub(crate) key: String,
+    #[pyo3(get, set)]
+    pub(crate) file_path: String,
+}
+
+#[pymethods]
+impl PutObjectFromFileInput {
+    #[new]
+    #[pyo3(signature = (bucket, key, file_path))]
+    pub fn new(bucket: &str, key: &str, file_path: &str) -> PyResult<Self> {
+        Ok(Self {
+            bucket: bucket.to_string(),
+            key: key.to_string(),
+            file_path: file_path.to_string(),
+        })
+    }
+}
+
+#[pyclass(name = "PutObjectOutput")]
+pub struct PutObjectOutput {
+    #[pyo3(get)]
+    pub(crate) request_id: String,
+    #[pyo3(get)]
+    pub(crate) status_code: isize,
+    #[pyo3(get)]
+    pub(crate) header: HashMap<String, String>,
+    #[pyo3(get)]
+    pub(crate) etag: String,
+    #[pyo3(get)]
+    pub(crate) version_id: String,
+    #[pyo3(get)]
+    pub(crate) hash_crc64ecma: u64,
+}

{tosnativeclient-1.0.0 → tosnativeclient-1.0.2}/src/write_stream.rs

@@ -1,5 +1,5 @@
 use crate::tos_client::InnerTosClient;
-use crate::tos_error::map_error;
+use crate::tos_error::map_tos_error;
 use async_channel::{Receiver, Sender};
 use futures_util::future::join_all;
 use pyo3::{pyclass, pymethods, PyRefMut, PyResult};
@@ -52,7 +52,7 @@ impl WriteStream {
             .py()
             .allow_threads(|| runtime.block_on(async move { writer.write(data, offset).await }))
         {
-            Err(ex) => Err(map_error(ex)),
+            Err(ex) => Err(map_tos_error(ex)),
             Ok(written) => {
                 slf.offset += written;
                 Ok(written)
@@ -67,14 +67,14 @@ impl WriteStream {
             .py()
             .allow_threads(|| runtime.block_on(async move { writer.release().await }))
         {
-            Err(ex) => Err(map_error(ex)),
+            Err(ex) => Err(map_tos_error(ex)),
             Ok(_) => Ok(()),
         }
     }
 }
 
 impl WriteStream {
-    pub(crate) fn new(
+    pub(crate) async fn new(
         client: Arc<InnerTosClient>,
         runtime: Arc<Runtime>,
         bucket: String,
@@ -93,10 +93,8 @@ impl WriteStream {
         let _key = key.clone();
         let _storage_class = storage_class.clone();
 
-        let object_writer = runtime.block_on(async move {
-            ObjectWriter::new(client, _bucket, _key, _storage_class, part_size).await
-        })?;
-
+        let object_writer =
+            ObjectWriter::new(client, _bucket, _key, _storage_class, part_size).await?;
         Ok(Self {
             object_writer: Arc::new(object_writer),
             runtime,

tosnativeclient-1.0.0/python/tosnativeclient/__init__.py

@@ -1,4 +0,0 @@
-from .tosnativeclient import TosClient, ListStream, ListObjectsResult, TosObject, TosError, TosException
-
-__all__ = ['TosClient', 'ListStream', 'ListObjectsResult', 'TosObject', 'TosError',
-           'TosException']

tosnativeclient-1.0.0/python/tosnativeclient/tosnativeclient.pyi

@@ -1,91 +0,0 @@
-from typing import List
-
-from typing import Optional
-
-
-class TosClient(object):
-    region: str
-    endpoint: str
-    ak: str
-    sk: str
-    part_size: int
-    max_retry_count: int
-    max_prefetch_tasks: int
-
-    def __init__(self, region: str, endpoint: str, ak: str = '', sk: str = '', part_size: int = 8388608,
-                 max_retry_count: int = 3, max_prefetch_tasks: int = 3, directives: str = '', directory: str = '',
-                 file_name_prefix: str = '', shared_prefetch_tasks: int = 20):
-        ...
-
-    def list_objects(self, bucket: str, prefix: str = '', max_keys: int = 1000, delimiter: str = '') -> ListStream:
-        ...
-
-    def head_object(self, bucket: str, key: str) -> TosObject:
-        ...
-
-    def get_object(self, bucket: str, key: str, etag: str, size: int) -> ReadStream:
-        ...
-
-    def put_object(self, bucket: str, key: str, storage_class: str = '') -> WriteStream:
-        ...
-
-
-class ListStream(object):
-    bucket: str
-    prefix: str
-    delimiter: str
-    max_keys: int
-
-    def __iter__(self) -> ListStream: ...
-
-    def __next__(self) -> ListObjectsResult: ...
-
-    def close(self) -> None: ...
-
-
-class ListObjectsResult(object):
-    contents: List[TosObject]
-    common_prefixes: List[str]
-
-
-class TosObject(object):
-    bucket: str
-    key: str
-    size: int
-    etag: str
-
-
-class ReadStream(object):
-    bucket: str
-    key: str
-    size: int
-    etag: str
-
-    def read(self, offset: int, length: int) -> bytes:
-        ...
-
-    def close(self) -> None:
-        ...
-
-
-class WriteStream(object):
-    bucket: str
-    key: str
-    storage_class: Optional[str]
-
-    def write(self, data: bytes) -> int:
-        ...
-
-    def close(self) -> None:
-        ...
-
-
-class TosError(object):
-    message: str
-    status_code: Optional[int]
-    ec: str
-    request_id: str
-
-
-class TosException(Exception):
-    args: List[TosError]