tosnativeclient 1.0.5__tar.gz → 1.0.7__tar.gz

{tosnativeclient-1.0.5 → tosnativeclient-1.0.7}/Cargo.lock

@@ -1521,7 +1521,7 @@ dependencies = [
 
 [[package]]
 name = "tosnativeclient"
-version = "1.0.5"
+version = "1.0.7"
 dependencies = [
  "arc-swap",
  "async-channel",

{tosnativeclient-1.0.5 → tosnativeclient-1.0.7}/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "tosnativeclient"
-version = "1.0.5"
+version = "1.0.7"
 edition = "2021"
 publish = false
 authors = ["xiangshijian"]

{tosnativeclient-1.0.5 → tosnativeclient-1.0.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tosnativeclient
-Version: 1.0.5
+Version: 1.0.7
 Classifier: Programming Language :: Rust
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy

{tosnativeclient-1.0.5 → tosnativeclient-1.0.7}/src/list_stream.rs

@@ -3,16 +3,18 @@ use crate::tos_error::{map_error_from_string, map_tos_error};
 use crate::tos_model::ListObjectsResult;
 use arc_swap::ArcSwap;
 use async_channel::Receiver;
+use futures_util::AsyncWriteExt;
 use pyo3::types::PyTuple;
 use pyo3::{pyclass, pymethods, Bound, IntoPyObject, PyRef, PyRefMut, PyResult, Python};
 use std::sync::atomic::{AtomicI8, Ordering};
 use std::sync::{Arc, RwLock};
 use tokio::runtime::Runtime;
+use tokio::task::JoinHandle;
 use ve_tos_rust_sdk::asynchronous::object::ObjectAPI;
 use ve_tos_rust_sdk::error::TosError;
 use ve_tos_rust_sdk::object::{ListObjectsType2Input, ListObjectsType2Output};
 
-const DEFAULT_TASK_COUNT: usize = 5;
+const DEFAULT_TASK_COUNT: usize = 10;
 #[pyclass(name = "ListStream", module = "tosnativeclient")]
 pub struct ListStream {
     client: Arc<InnerTosClient>,
@@ -58,13 +60,19 @@ impl ListStream {
         slf.next_page(pg.as_ref(), slf.py())
     }
 
-    pub fn close(&self) {
-        if let Ok(_) = self
+    pub fn close(slf: PyRef<'_, Self>) {
+        if let Ok(_) = slf
             .closed
             .compare_exchange(0, 1, Ordering::AcqRel, Ordering::Relaxed)
         {
-            if let Some(pg) = self.paginator.read().unwrap().as_ref() {
+            if let Some(pg) = slf.paginator.write().unwrap().as_mut() {
                 pg.receiver.close();
+                let runtime = slf.runtime.clone();
+                slf.py().allow_threads(|| {
+                    runtime.block_on(async {
+                        pg.close().await;
+                    })
+                });
             }
         }
     }
@@ -124,7 +132,7 @@ impl ListStream {
         if self.start_after != "" {
             input.set_start_after(self.start_after.as_str());
         }
-        py.allow_threads(|| {
+        let wait_list_background = py.allow_threads(|| {
             self.runtime.spawn(async move {
                 let mut need_break = false;
                 if input.delimiter() == "" {
@@ -181,7 +189,7 @@ impl ListStream {
                         }
                     }
                 }
-            });
+            })
         });
         Some(Paginator {
             is_end: ArcSwap::new(Arc::new(false)),
@@ -189,6 +197,7 @@ impl ListStream {
             current_prefix: ArcSwap::new(Arc::new(self.prefix.clone())),
             current_continuation_token: ArcSwap::new(Arc::new(self.continuation_token.clone())),
             receiver,
+            wait_list_background: Some(wait_list_background),
         })
     }
 
@@ -228,6 +237,7 @@ pub(crate) struct Paginator {
     current_prefix: ArcSwap<String>,
     current_continuation_token: ArcSwap<String>,
     receiver: Receiver<(bool, Result<ListObjectsType2Output, TosError>)>,
+    wait_list_background: Option<JoinHandle<()>>,
 }
 
 impl Paginator {
@@ -244,6 +254,13 @@ impl Paginator {
     fn current_continuation_token(&self) -> String {
         self.current_continuation_token.load().to_string()
     }
+
+    async fn close(&mut self) {
+        if let Some(wait_list_background) = self.wait_list_background.take() {
+            let _ = wait_list_background.await;
+        }
+    }
+
     async fn next_page(&self) -> Result<ListObjectsType2Output, TosError> {
         if let Some(e) = self.last_err.load().as_ref() {
             return Err(e.clone());

{tosnativeclient-1.0.5 → tosnativeclient-1.0.7}/src/read_stream.rs

@@ -11,6 +11,7 @@ use std::sync::atomic::{AtomicI8, AtomicIsize, Ordering};
 use std::sync::Arc;
 use tokio::runtime::{Handle, Runtime};
 use tokio::sync::{Mutex, MutexGuard};
+use tokio::task::JoinHandle;
 use tracing::log::{error, warn};
 use ve_tos_rust_sdk::asynchronous::object::ObjectAPI;
 use ve_tos_rust_sdk::error::TosError;
@@ -341,9 +342,9 @@ impl ObjectFetcher {
             return None;
         }
 
-        let task = self.new_fetch_task(fc);
+        let mut task = self.new_fetch_task(fc);
         self.revise_next_request_offset_and_size(fc, task.size);
-        task.async_fetch(self.client.clone(), runtime);
+        task.wait_async_fetch = Some(task.async_fetch(self.client.clone(), runtime));
         Some(task)
     }
 
@@ -512,6 +513,7 @@ impl ObjectFetcher {
             chunk_queue: Arc::new(ChunkQueue::new(self.calc_chunk_queue_size() as usize)),
             last_chunk: ChunkHolder::new(None),
             closed: self.closed.clone(),
+            wait_async_fetch: None,
         }
     }
 
@@ -537,6 +539,7 @@ impl ObjectFetcher {
             chunk_queue: Arc::new(ChunkQueue::new(chunks.len())),
             last_chunk: ChunkHolder { inner: None },
             closed: self.closed.clone(),
+            wait_async_fetch: None,
         };
 
         for chunk in chunks {
@@ -653,6 +656,7 @@ struct FetchTask {
     chunk_queue: Arc<ChunkQueue>,
     last_chunk: ChunkHolder,
     closed: Arc<AtomicI8>,
+    wait_async_fetch: Option<JoinHandle<()>>,
 }
 
 impl FetchTask {
@@ -666,12 +670,12 @@ impl FetchTask {
         self.start_offset + self.fetched_size.load(Ordering::Acquire)
     }
 
-    fn async_fetch(&self, client: Arc<InnerTosClient>, runtime: Arc<Runtime>) {
+    fn async_fetch(&self, client: Arc<InnerTosClient>, runtime: Arc<Runtime>) -> JoinHandle<()> {
         let fetch_task_context = FetchTaskContext::new(self, client);
         runtime.spawn(async move {
             fetch_task_context.fetch_from_server().await;
             fetch_task_context.chunk_queue.close();
-        });
+        })
     }
 
     async fn read(&mut self, length: isize) -> Option<Chunk> {
@@ -716,6 +720,9 @@ impl FetchTask {
             self.shared_prefetch_task.fetch_add(-1, Ordering::Release);
             self.pcontext.release_shared_prefetch_task();
         }
+        if let Some(wait_execute) = self.wait_async_fetch.take() {
+            let _ = wait_execute.await;
+        }
     }
 }
 

{tosnativeclient-1.0.5 → tosnativeclient-1.0.7}/src/tos_client.rs

@@ -192,7 +192,7 @@ impl TosClient {
             runtime.block_on(async move {
                 match client.head_object(&input).await {
                     Err(ex) => Err(map_tos_error(ex)),
-                    Ok(output) => Ok(TosObject::new(input.bucket(), input.key(), output)),
+                    Ok(output) => Ok(TosObject::inner_new(input.bucket(), input.key(), output)),
                 }
             })
         })

{tosnativeclient-1.0.5 → tosnativeclient-1.0.7}/src/tos_model.rs

@@ -1,4 +1,5 @@
-use pyo3::pyclass;
+use pyo3::types::PyTuple;
+use pyo3::{pyclass, pymethods, Bound, IntoPyObject, PyRef, PyResult};
 use ve_tos_rust_sdk::object::{HeadObjectOutput, ListObjectsType2Output};
 
 #[pyclass(name = "ListObjectsResult", module = "tosnativeclient")]
@@ -44,9 +45,32 @@ pub struct TosObject {
     #[pyo3(get)]
     pub(crate) etag: String,
 }
+#[pymethods]
+impl TosObject {
+    #[new]
+    #[pyo3(signature = (bucket, key, size, etag))]
+    pub fn new(bucket: String, key: String, size: isize, etag: String) -> Self {
+        Self {
+            bucket,
+            key,
+            size,
+            etag,
+        }
+    }
+    pub fn __getnewargs__(slf: PyRef<'_, Self>) -> PyResult<Bound<'_, PyTuple>> {
+        let py = slf.py();
+        let state = [
+            slf.bucket.clone().into_pyobject(py)?.into_any(),
+            slf.key.clone().into_pyobject(py)?.into_any(),
+            slf.size.into_pyobject(py)?.into_any(),
+            slf.etag.clone().into_pyobject(py)?.into_any(),
+        ];
+        PyTuple::new(py, state)
+    }
+}
 
 impl TosObject {
-    pub(crate) fn new(bucket: &str, key: &str, output: HeadObjectOutput) -> Self {
+    pub(crate) fn inner_new(bucket: &str, key: &str, output: HeadObjectOutput) -> Self {
         Self {
             bucket: bucket.to_string(),
             key: key.to_string(),