tosnativeclient 1.0.4__tar.gz → 1.0.6__tar.gz

{tosnativeclient-1.0.4 → tosnativeclient-1.0.6}/Cargo.lock

@@ -1521,7 +1521,7 @@ dependencies = [
 
 [[package]]
 name = "tosnativeclient"
-version = "1.0.4"
+version = "1.0.6"
 dependencies = [
  "arc-swap",
  "async-channel",

{tosnativeclient-1.0.4 → tosnativeclient-1.0.6}/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "tosnativeclient"
-version = "1.0.4"
+version = "1.0.6"
 edition = "2021"
 publish = false
 authors = ["xiangshijian"]

{tosnativeclient-1.0.4 → tosnativeclient-1.0.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tosnativeclient
-Version: 1.0.4
+Version: 1.0.6
 Classifier: Programming Language :: Rust
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy

{tosnativeclient-1.0.4 → tosnativeclient-1.0.6}/python/tosnativeclient/tosnativeclient.pyi

@@ -31,6 +31,8 @@ class ListStream(object):
     prefix: str
     delimiter: str
     max_keys: int
+    continuation_token: str
+    start_after: str
 
     def __iter__(self) -> ListStream: ...
 
@@ -38,6 +40,10 @@ class ListStream(object):
 
     def close(self) -> None: ...
 
+    def current_prefix(self) -> Optional[str]: ...
+
+    def current_continuation_token(self) -> Optional[str]: ...
+
 
 class ReadStream(object):
     bucket: str
@@ -82,7 +88,8 @@ class TosClient(object):
                  file_name_prefix: str = '', shared_prefetch_tasks: int = 20):
         ...
 
-    def list_objects(self, bucket: str, prefix: str = '', max_keys: int = 1000, delimiter: str = '') -> ListStream:
+    def list_objects(self, bucket: str, prefix: str = '', max_keys: int = 1000, delimiter: str = '',
+                     continuation_token: str = '', start_after: str = '') -> ListStream:
         ...
 
     def head_object(self, bucket: str, key: str) -> TosObject:

{tosnativeclient-1.0.4 → tosnativeclient-1.0.6}/src/list_stream.rs

@@ -27,6 +27,10 @@ pub struct ListStream {
     delimiter: String,
     #[pyo3(get)]
     max_keys: isize,
+    #[pyo3(get)]
+    continuation_token: String,
+    #[pyo3(get)]
+    start_after: String,
 }
 
 #[pymethods]
@@ -65,15 +69,20 @@ impl ListStream {
         }
     }
 
-    pub fn __getnewargs__(slf: PyRef<'_, Self>) -> PyResult<Bound<'_, PyTuple>> {
-        let py = slf.py();
-        let state = [
-            slf.bucket.clone().into_pyobject(py)?.into_any(),
-            slf.prefix.clone().into_pyobject(py)?.into_any(),
-            slf.delimiter.clone().into_pyobject(py)?.into_any(),
-            slf.max_keys.clone().into_pyobject(py)?.into_any(),
-        ];
-        PyTuple::new(py, state)
+    pub fn current_prefix(&self) -> PyResult<Option<String>> {
+        let pg = self.paginator.read().unwrap();
+        match pg.as_ref() {
+            None => Ok(None),
+            Some(pg) => Ok(Some(pg.current_prefix())),
+        }
+    }
+
+    pub fn current_continuation_token(&self) -> PyResult<Option<String>> {
+        let pg = self.paginator.read().unwrap();
+        match pg.as_ref() {
+            None => Ok(None),
+            Some(pg) => Ok(Some(pg.current_continuation_token())),
+        }
     }
 }
 
@@ -85,6 +94,8 @@ impl ListStream {
         prefix: String,
         delimiter: String,
         max_keys: isize,
+        continuation_token: String,
+        start_after: String,
     ) -> Self {
         Self {
             client,
@@ -95,6 +106,8 @@ impl ListStream {
             prefix,
             delimiter,
             max_keys,
+            continuation_token,
+            start_after,
         }
     }
 
@@ -105,7 +118,12 @@ impl ListStream {
         input.set_prefix(self.prefix.as_str());
         input.set_max_keys(self.max_keys);
         input.set_delimiter(self.delimiter.as_str());
-
+        if self.continuation_token != "" {
+            input.set_continuation_token(self.continuation_token.as_str());
+        }
+        if self.start_after != "" {
+            input.set_start_after(self.start_after.as_str());
+        }
         py.allow_threads(|| {
             self.runtime.spawn(async move {
                 let mut need_break = false;
@@ -169,6 +187,7 @@ impl ListStream {
             is_end: ArcSwap::new(Arc::new(false)),
             last_err: ArcSwap::new(Arc::new(None)),
             current_prefix: ArcSwap::new(Arc::new(self.prefix.clone())),
+            current_continuation_token: ArcSwap::new(Arc::new(self.continuation_token.clone())),
             receiver,
         })
     }
@@ -207,6 +226,7 @@ pub(crate) struct Paginator {
     is_end: ArcSwap<bool>,
     last_err: ArcSwap<Option<TosError>>,
     current_prefix: ArcSwap<String>,
+    current_continuation_token: ArcSwap<String>,
     receiver: Receiver<(bool, Result<ListObjectsType2Output, TosError>)>,
 }
 
@@ -218,8 +238,11 @@ impl Paginator {
         Ok(!*self.is_end.load().as_ref())
     }
 
-    fn current_prefix(&self) -> Arc<String> {
-        self.current_prefix.load().clone()
+    fn current_prefix(&self) -> String {
+        self.current_prefix.load().to_string()
+    }
+    fn current_continuation_token(&self) -> String {
+        self.current_continuation_token.load().to_string()
     }
     async fn next_page(&self) -> Result<ListObjectsType2Output, TosError> {
         if let Some(e) = self.last_err.load().as_ref() {
@@ -248,6 +271,8 @@ impl Paginator {
                 Ok(output) => {
                     self.current_prefix
                         .store(Arc::new(output.prefix().to_string()));
+                    self.current_continuation_token
+                        .store(Arc::new(output.continuation_token().to_string()));
                     if is_end {
                         self.is_end.store(Arc::new(true));
                     }

{tosnativeclient-1.0.4 → tosnativeclient-1.0.6}/src/read_stream.rs

@@ -11,6 +11,7 @@ use std::sync::atomic::{AtomicI8, AtomicIsize, Ordering};
 use std::sync::Arc;
 use tokio::runtime::{Handle, Runtime};
 use tokio::sync::{Mutex, MutexGuard};
+use tokio::task::JoinHandle;
 use tracing::log::{error, warn};
 use ve_tos_rust_sdk::asynchronous::object::ObjectAPI;
 use ve_tos_rust_sdk::error::TosError;
@@ -66,17 +67,6 @@ impl ReadStream {
         slf.py()
             .allow_threads(|| runtime.block_on(async move { fetcher.close().await }))
     }
-
-    pub fn __getnewargs__(slf: PyRef<'_, Self>) -> PyResult<Bound<'_, PyTuple>> {
-        let py = slf.py();
-        let state = [
-            slf.bucket.clone().into_pyobject(py)?.into_any(),
-            slf.key.clone().into_pyobject(py)?.into_any(),
-            slf.etag.clone().into_pyobject(py)?.into_any(),
-            slf.size.clone().into_pyobject(py)?.into_any(),
-        ];
-        PyTuple::new(py, state)
-    }
 }
 
 impl ReadStream {
@@ -128,6 +118,7 @@ impl ReadStream {
         Self {
             object_fetcher: Arc::new(ObjectFetcher {
                 client,
+                runtime: runtime.clone(),
                 closed: Arc::new(AtomicI8::new(0)),
                 pc,
                 fc: Mutex::new(fc),
@@ -160,6 +151,7 @@ impl ReadStream {
 
 struct ObjectFetcher {
     client: Arc<InnerTosClient>,
+    runtime: Arc<Runtime>,
     closed: Arc<AtomicI8>,
     pc: PrefetchConfig,
     fc: Mutex<FetchContext>,
@@ -306,7 +298,7 @@ impl ObjectFetcher {
                 if fc.has_tasks() {
                     fc.current = fc.pop_task()
                 } else {
-                    fc.current = self.start_fetch_task(fc);
+                    fc.current = self.start_fetch_task(fc, self.runtime.clone());
                 }
             }
             Some(current) => {
@@ -315,17 +307,17 @@ impl ObjectFetcher {
                     if fc.has_tasks() {
                         fc.current = fc.pop_task()
                     } else {
-                        fc.current = self.start_fetch_task(fc);
+                        fc.current = self.start_fetch_task(fc, self.runtime.clone());
                     }
                 } else if fc.exceed_sequential_read_threshold() {
                     if fc.can_add_task(self.pc.max_prefetch_tasks) {
-                        if let Some(next) = self.start_fetch_task(fc) {
+                        if let Some(next) = self.start_fetch_task(fc, self.runtime.clone()) {
                             fc.push_task(next, false);
                         }
                     } else if fc.can_steal_shared_task(self.pc.shared_prefetch_task_limit)
                         && fc.pcontext.try_steal_shared_prefetch_task()
                     {
-                        match self.start_fetch_task(fc) {
+                        match self.start_fetch_task(fc, self.runtime.clone()) {
                             Some(mut next) => {
                                 next.is_shared = true;
                                 fc.steal_shared_task();
@@ -341,14 +333,18 @@ impl ObjectFetcher {
         }
     }
 
-    fn start_fetch_task(&self, fc: &mut MutexGuard<'_, FetchContext>) -> Option<FetchTask> {
+    fn start_fetch_task(
+        &self,
+        fc: &mut MutexGuard<'_, FetchContext>,
+        runtime: Arc<Runtime>,
+    ) -> Option<FetchTask> {
         if fc.next_request_offset >= self.om.size {
             return None;
         }
 
-        let task = self.new_fetch_task(fc);
+        let mut task = self.new_fetch_task(fc);
         self.revise_next_request_offset_and_size(fc, task.size);
-        task.async_fetch(self.client.clone());
+        task.wait_async_fetch = Some(task.async_fetch(self.client.clone(), runtime));
         Some(task)
     }
 
@@ -517,6 +513,7 @@ impl ObjectFetcher {
             chunk_queue: Arc::new(ChunkQueue::new(self.calc_chunk_queue_size() as usize)),
             last_chunk: ChunkHolder::new(None),
             closed: self.closed.clone(),
+            wait_async_fetch: None,
         }
     }
 
@@ -542,6 +539,7 @@ impl ObjectFetcher {
             chunk_queue: Arc::new(ChunkQueue::new(chunks.len())),
             last_chunk: ChunkHolder { inner: None },
             closed: self.closed.clone(),
+            wait_async_fetch: None,
         };
 
         for chunk in chunks {
@@ -658,6 +656,7 @@ struct FetchTask {
     chunk_queue: Arc<ChunkQueue>,
     last_chunk: ChunkHolder,
     closed: Arc<AtomicI8>,
+    wait_async_fetch: Option<JoinHandle<()>>,
 }
 
 impl FetchTask {
@@ -671,12 +670,12 @@ impl FetchTask {
         self.start_offset + self.fetched_size.load(Ordering::Acquire)
     }
 
-    fn async_fetch(&self, client: Arc<InnerTosClient>) {
+    fn async_fetch(&self, client: Arc<InnerTosClient>, runtime: Arc<Runtime>) -> JoinHandle<()> {
         let fetch_task_context = FetchTaskContext::new(self, client);
-        Handle::current().spawn(async move {
+        runtime.spawn(async move {
             fetch_task_context.fetch_from_server().await;
             fetch_task_context.chunk_queue.close();
-        });
+        })
     }
 
     async fn read(&mut self, length: isize) -> Option<Chunk> {
@@ -721,6 +720,9 @@ impl FetchTask {
             self.shared_prefetch_task.fetch_add(-1, Ordering::Release);
             self.pcontext.release_shared_prefetch_task();
         }
+        if let Some(wait_execute) = self.wait_async_fetch.take() {
+            let _ = wait_execute.await;
+        }
     }
 }
 

{tosnativeclient-1.0.4 → tosnativeclient-1.0.6}/src/tos_client.rs

@@ -163,13 +163,15 @@ impl TosClient {
         })
     }
 
-    #[pyo3(signature = (bucket, prefix=String::from(""), max_keys=1000, delimiter=String::from("")))]
+    #[pyo3(signature = (bucket, prefix=String::from(""), max_keys=1000, delimiter=String::from(""), continuation_token=String::from(""), start_after=String::from("")))]
     pub fn list_objects(
         &self,
         bucket: String,
         prefix: String,
         max_keys: isize,
         delimiter: String,
+        continuation_token: String,
+        start_after: String,
     ) -> ListStream {
         ListStream::new(
             self.rclient.clone(),
@@ -178,6 +180,8 @@ impl TosClient {
             prefix,
             delimiter,
             max_keys,
+            continuation_token,
+            start_after,
         )
     }
     pub fn head_object(slf: PyRef<'_, Self>, bucket: String, key: String) -> PyResult<TosObject> {

{tosnativeclient-1.0.4 → tosnativeclient-1.0.6}/src/tos_raw_client.rs

@@ -228,10 +228,10 @@ impl TosRawClient {
             slf.endpoint.clone().into_pyobject(py)?.into_any(),
             slf.ak.clone().into_pyobject(py)?.into_any(),
             slf.sk.clone().into_pyobject(py)?.into_any(),
-            slf.connection_timeout.clone().into_pyobject(py)?.into_any(),
-            slf.request_timeout.clone().into_pyobject(py)?.into_any(),
-            slf.max_connections.clone().into_pyobject(py)?.into_any(),
-            slf.max_retry_count.clone().into_pyobject(py)?.into_any(),
+            slf.connection_timeout.into_pyobject(py)?.into_any(),
+            slf.request_timeout.into_pyobject(py)?.into_any(),
+            slf.max_connections.into_pyobject(py)?.into_any(),
+            slf.max_retry_count.into_pyobject(py)?.into_any(),
         ];
         PyTuple::new(py, state)
     }

{tosnativeclient-1.0.4 → tosnativeclient-1.0.6}/src/write_stream.rs

@@ -72,16 +72,6 @@ impl WriteStream {
             Ok(_) => Ok(()),
         }
     }
-
-    pub fn __getnewargs__(slf: PyRef<'_, Self>) -> PyResult<Bound<'_, PyTuple>> {
-        let py = slf.py();
-        let state = [
-            slf.bucket.clone().into_pyobject(py)?.into_any(),
-            slf.key.clone().into_pyobject(py)?.into_any(),
-            slf.storage_class.clone().into_pyobject(py)?.into_any(),
-        ];
-        PyTuple::new(py, state)
-    }
 }
 
 impl WriteStream {
@@ -104,8 +94,15 @@ impl WriteStream {
         let _key = key.clone();
         let _storage_class = storage_class.clone();
 
-        let object_writer =
-            ObjectWriter::new(client, _bucket, _key, _storage_class, part_size).await?;
+        let object_writer = ObjectWriter::new(
+            client,
+            runtime.clone(),
+            _bucket,
+            _key,
+            _storage_class,
+            part_size,
+        )
+        .await?;
         Ok(Self {
             object_writer: Arc::new(object_writer),
             runtime,
@@ -145,6 +142,7 @@ struct ObjectWriter {
 impl ObjectWriter {
     async fn new(
         client: Arc<InnerTosClient>,
+        runtime: Arc<Runtime>,
         bucket: String,
         key: String,
         storage_class: Option<String>,
@@ -155,7 +153,15 @@ impl ObjectWriter {
             key,
             storage_class,
         });
-        let ou = ObjectUploader::new(client, wp, part_size, 0, Arc::new(AtomicI8::new(0))).await?;
+        let ou = ObjectUploader::new(
+            client,
+            runtime,
+            wp,
+            part_size,
+            0,
+            Arc::new(AtomicI8::new(0)),
+        )
+        .await?;
         Ok(Self {
             ctx: Mutex::new((ou, 0)),
             closed: Arc::new(AtomicI8::new(0)),
@@ -274,6 +280,7 @@ struct WriteParam {
 struct ObjectUploader {
     next_write_offset: isize,
     uc: Arc<UploadContext>,
+    runtime: Arc<Runtime>,
     current: Option<Part>,
     part_size: isize,
     max_size: isize,
@@ -288,6 +295,7 @@ struct ObjectUploader {
 impl ObjectUploader {
     async fn new(
         client: Arc<InnerTosClient>,
+        runtime: Arc<Runtime>,
         wp: Arc<WriteParam>,
         part_size: isize,
         next_write_offset: isize,
@@ -304,6 +312,7 @@ impl ObjectUploader {
         let mut ou = Self {
             next_write_offset,
             uc: Arc::new(UploadContext::new(created, wp, client)),
+            runtime,
             current: None,
             part_size,
             max_size,
@@ -485,7 +494,7 @@ impl ObjectUploader {
         let dp = self.dp.clone();
         let st = self.st.clone();
         let uc = self.uc.clone();
-        self.wait_dispatch = Some(Handle::current().spawn(async move {
+        self.wait_dispatch = Some(self.runtime.spawn(async move {
             loop {
                 match dp.pull().await {
                     None => return,
@@ -510,12 +519,13 @@ impl ObjectUploader {
     }
 
     async fn execute(&mut self) {
+        let runtime = self.runtime.clone();
         let dp = self.dp.clone();
         let st = self.st.clone();
         let ta = self.ta.clone();
         let uc = self.uc.clone();
         let mu_ctx = self.mu_ctx.clone();
-        self.wait_execute = Some(Handle::current().spawn(async move {
+        self.wait_execute = Some(self.runtime.spawn(async move {
             let mut wait_async_uploads = Vec::with_capacity(16);
             loop {
                 match st.pull().await {
@@ -529,6 +539,7 @@ impl ObjectUploader {
                         wait_async_uploads.push(
                             uc.clone()
                                 .async_upload(
+                                    runtime.clone(),
                                     dp.clone(),
                                     st.clone(),
                                     ta.clone(),
@@ -600,6 +611,7 @@ impl UploadContext {
 
     async fn async_upload(
         self: Arc<Self>,
+        runtime: Arc<Runtime>,
         dp: Arc<Dispatcher>,
         st: Arc<Store>,
         ta: Arc<TokenAcquirer>,
@@ -607,7 +619,7 @@ impl UploadContext {
         mut si: StoreItem,
     ) -> JoinHandle<()> {
         let aborted = self.aborted.clone();
-        Handle::current().spawn(async move {
+        runtime.spawn(async move {
             let _ = ta.acquire().await;
             if aborted.load(Ordering::Acquire) == 1 {
                 si.release();