tostorchconnector 1.0.9__tar.gz → 1.1.0__tar.gz

{tostorchconnector-1.0.9/tostorchconnector.egg-info → tostorchconnector-1.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tostorchconnector
-Version: 1.0.9
+Version: 1.1.0
 Summary: TOS connector integration for PyTorch
 Author-email: xiangshijian <xiangshijian@bytedance.com>
 Classifier: Development Status :: 4 - Beta
@@ -19,7 +19,7 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: torch>=2.0
 Requires-Dist: tos>=2.8.0
-Requires-Dist: tosnativeclient>=1.0.6
+Requires-Dist: tosnativeclient>=1.0.7
 Dynamic: license-file
 
 # TOS Connector for pytorch

{tostorchconnector-1.0.9 → tostorchconnector-1.1.0}/pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "tostorchconnector"
-version = "1.0.9"
+version = "1.1.0"
 description = "TOS connector integration for PyTorch"
 authors = [{ name = "xiangshijian", email = "xiangshijian@bytedance.com" }]
 requires-python = ">=3.8,<3.14"
@@ -26,7 +26,7 @@ classifiers = [
 dependencies = [
     "torch >= 2.0",
     "tos>=2.8.0",
-    "tosnativeclient >= 1.0.6"
+    "tosnativeclient >= 1.0.7"
 ]
 
 [tool.setuptools.packages]

{tostorchconnector-1.0.9 → tostorchconnector-1.1.0}/tostorchconnector/tos_common.py

@@ -1,6 +1,7 @@
 import logging
 from typing import Union, Iterator, Tuple, Optional
 
+from tosnativeclient import TosObject
 from . import TosObjectReader
 from .tos_client import TosClient
 from .tos_object_meta import TosObjectMeta
@@ -14,7 +15,7 @@ class TosObjectIterable(object):
         self._prefix = prefix
         self._client = client
 
-    def __iter__(self) -> Iterator[TosObjectMeta]:
+    def __iter__(self) -> Iterator[TosObjectMeta | TosObject]:
         return iter(TosObjectIterator(self._bucket, self._prefix, self._client))
 
 
@@ -35,10 +36,10 @@ class TosObjectIterator(object):
         if self._list_stream is not None:
             self._list_stream.close()
 
-    def __iter__(self) -> Iterator[TosObjectMeta]:
+    def __iter__(self) -> Iterator[TosObjectMeta | TosObject]:
         return self
 
-    def __next__(self) -> TosObjectMeta:
+    def __next__(self) -> TosObjectMeta | TosObject:
         if self._client.use_native_client:
             if self._list_stream is None:
                 self._list_stream = self._client.gen_list_stream(self._bucket, self._prefix, max_keys=1000,
@@ -46,15 +47,17 @@ class TosObjectIterator(object):
                                                                  continuation_token=self._continuation_token)
 
             if self._object_metas is None or self._index >= len(self._object_metas):
-                self._object_metas = []
+                self._object_metas = None
                 self._index = 0
                 while 1:
-                    objects = next(self._list_stream)
+                    try:
+                        objects = next(self._list_stream)
+                    except:
+                        self.close()
+                        raise
                     self._continuation_token = self._list_stream.current_continuation_token()
-                    for content in objects.contents:
-                        self._object_metas.append(
-                            TosObjectMeta(content.bucket, content.key, content.size, content.etag))
-                    if len(self._object_metas) > 0:
+                    self._object_metas = objects.contents
+                    if self._object_metas is not None and len(self._object_metas) > 0:
                         break
 
             object_meta = self._object_metas[self._index]
@@ -104,12 +107,12 @@ def default_trans(obj: TosObjectReader) -> TosObjectReader:
     return obj
 
 
-def gen_dataset_from_urls(urls: Union[str, Iterator[str]], _: TosClient) -> Iterator[TosObjectMeta]:
+def gen_dataset_from_urls(urls: Union[str, Iterator[str]], _: TosClient) -> Iterator[TosObjectMeta | TosObject]:
     if isinstance(urls, str):
         urls = [urls]
     return (TosObjectMeta(bucket, key) for bucket, key in [parse_tos_url(url) for url in urls])
 
 
-def gen_dataset_from_prefix(prefix: str, client: TosClient) -> Iterator[TosObjectMeta]:
+def gen_dataset_from_prefix(prefix: str, client: TosClient) -> Iterator[TosObjectMeta | TosObject]:
     bucket, prefix = parse_tos_url(prefix)
     return iter(TosObjectIterable(bucket, prefix, client))

{tostorchconnector-1.0.9 → tostorchconnector-1.1.0}/tostorchconnector/tos_iterable_dataset.py

@@ -4,6 +4,7 @@ from typing import Iterator, Any, Optional, Callable, Union
 
 import torch
 
+from tosnativeclient import TosObject
 from . import TosObjectReader
 from .tos_client import CredentialProvider, TosClientConfig, TosClient, TosLogConfig, ReaderType
 from .tos_common import default_trans, gen_dataset_from_urls, gen_dataset_from_prefix
@@ -14,7 +15,7 @@ log = logging.getLogger(__name__)
 
 class TosIterableDataset(torch.utils.data.IterableDataset):
     def __init__(self, region: str,
-                 gen_dataset: Callable[[TosClient], Iterator[TosObjectMeta]],
+                 gen_dataset: Callable[[TosClient], Iterator[TosObjectMeta | TosObject]],
                  endpoint: Optional[str] = None,
                  transform: Callable[[TosObjectReader], Any] = default_trans,
                  cred: Optional[CredentialProvider] = None,
@@ -100,7 +101,7 @@ class TosIterableDataset(torch.utils.data.IterableDataset):
         )
         return map(self._trans_tos_object, part_dataset)
 
-    def _trans_tos_object(self, object_meta: TosObjectMeta) -> Any:
+    def _trans_tos_object(self, object_meta: TosObjectMeta | TosObject) -> Any:
         obj = self._client.get_object(object_meta.bucket, object_meta.key, object_meta.etag, object_meta.size,
                                       reader_type=self._reader_type, buffer_size=self._buffer_size)
         return self._trans(obj)

{tostorchconnector-1.0.9 → tostorchconnector-1.1.0}/tostorchconnector/tos_map_dataset.py

@@ -4,6 +4,7 @@ from typing import Any, Callable, Iterator, Optional, List, Union
 
 import torch
 
+from tosnativeclient import TosObject
 from . import TosObjectReader
 from .tos_client import CredentialProvider, TosClientConfig, TosClient, TosLogConfig, ReaderType
 from .tos_common import default_trans, gen_dataset_from_prefix, \
@@ -15,7 +16,7 @@ log = logging.getLogger(__name__)
 
 class TosMapDataset(torch.utils.data.Dataset):
     def __init__(self, region: str,
-                 gen_dataset: Callable[[TosClient], Iterator[TosObjectMeta]],
+                 gen_dataset: Callable[[TosClient], Iterator[TosObjectMeta | TosObject]],
                  endpoint: Optional[str] = None,
                  transform: Callable[[TosObjectReader], Any] = default_trans,
                  cred: Optional[CredentialProvider] = None,
@@ -71,7 +72,7 @@ class TosMapDataset(torch.utils.data.Dataset):
         return len(self._data_set)
 
     @property
-    def _data_set(self) -> List[TosObjectMeta]:
+    def _data_set(self) -> List[TosObjectMeta | TosObject]:
         if self._dataset is None:
             self._dataset = list(self._gen_dataset(self._client))
         assert self._dataset is not None

{tostorchconnector-1.0.9 → tostorchconnector-1.1.0/tostorchconnector.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tostorchconnector
-Version: 1.0.9
+Version: 1.1.0
 Summary: TOS connector integration for PyTorch
 Author-email: xiangshijian <xiangshijian@bytedance.com>
 Classifier: Development Status :: 4 - Beta
@@ -19,7 +19,7 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: torch>=2.0
 Requires-Dist: tos>=2.8.0
-Requires-Dist: tosnativeclient>=1.0.6
+Requires-Dist: tosnativeclient>=1.0.7
 Dynamic: license-file
 
 # TOS Connector for pytorch

tostorchconnector-1.1.0/tostorchconnector.egg-info/requires.txt

@@ -0,0 +1,3 @@
+torch>=2.0
+tos>=2.8.0
+tosnativeclient>=1.0.7

tostorchconnector-1.0.9/tostorchconnector.egg-info/requires.txt

@@ -1,3 +0,0 @@
-torch>=2.0
-tos>=2.8.0
-tosnativeclient>=1.0.6