tostorchconnector 1.0.6__tar.gz → 1.0.7__tar.gz

{tostorchconnector-1.0.6/tostorchconnector.egg-info → tostorchconnector-1.0.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tostorchconnector
-Version: 1.0.6
+Version: 1.0.7
 Summary: TOS connector integration for PyTorch
 Author-email: xiangshijian <xiangshijian@bytedance.com>
 Classifier: Development Status :: 4 - Beta
@@ -19,7 +19,7 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: torch>=2.0
 Requires-Dist: tos>=2.8.0
-Requires-Dist: tosnativeclient>=1.0.2
+Requires-Dist: tosnativeclient>=1.0.4
 Dynamic: license-file
 
 # TOS Connector for pytorch

{tostorchconnector-1.0.6 → tostorchconnector-1.0.7}/pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "tostorchconnector"
-version = "1.0.6"
+version = "1.0.7"
 description = "TOS connector integration for PyTorch"
 authors = [{ name = "xiangshijian", email = "xiangshijian@bytedance.com" }]
 requires-python = ">=3.8,<3.14"
@@ -26,7 +26,7 @@ classifiers = [
 dependencies = [
     "torch >= 2.0",
     "tos>=2.8.0",
-    "tosnativeclient >= 1.0.2"
+    "tosnativeclient >= 1.0.4"
 ]
 
 [tool.setuptools.packages]

{tostorchconnector-1.0.6 → tostorchconnector-1.0.7}/tests/test_tos_dataset.py

@@ -1,6 +1,8 @@
 import os
 import unittest
 
+from torch.utils.data import DataLoader
+
 from tostorchconnector import TosMapDataset, TosIterableDataset, TosCheckpoint
 from tostorchconnector.tos_client import CredentialProvider, ReaderType
 
@@ -31,6 +33,7 @@ class TestTosDataSet(unittest.TestCase):
         datasets = TosMapDataset.from_prefix(f'tos://{bucket}/prefix', region=region,
                                              endpoint=endpoint, cred=CredentialProvider(ak, sk),
                                              use_native_client=USE_NATIVE_CLIENT)
+
         for i in range(len(datasets)):
             item = datasets[i]
             print(item.bucket, item.key)

{tostorchconnector-1.0.6 → tostorchconnector-1.0.7}/tests/test_tosclient.py

@@ -1,4 +1,5 @@
 import os
+import pickle
 import unittest
 import uuid
 
@@ -6,6 +7,21 @@ from tosnativeclient import TosClient, TosException
 
 
 class TestTosClient(unittest.TestCase):
+    def test_pickle(self):
+        region = os.getenv('TOS_REGION')
+        endpoint = os.getenv('TOS_ENDPOINT')
+        ak = os.getenv('TOS_ACCESS_KEY')
+        sk = os.getenv('TOS_SECRET_KEY')
+        bucket = 'tos-pytorch-connector'
+        tos_client = TosClient(region, endpoint, ak, sk, directives='info', directory='logs',
+                               file_name_prefix='app.log')
+        pickled_tos_client = pickle.dumps(tos_client)
+        assert isinstance(pickled_tos_client, bytes)
+        unpickled_tos_client = pickle.loads(pickled_tos_client)
+
+        self._test_list_objects(bucket, unpickled_tos_client)
+        self._test_write_read_object(bucket, unpickled_tos_client)
+
     def test_list_objects(self):
         region = os.getenv('TOS_REGION')
         endpoint = os.getenv('TOS_ENDPOINT')
@@ -14,7 +30,20 @@ class TestTosClient(unittest.TestCase):
         bucket = 'tos-pytorch-connector'
         tos_client = TosClient(region, endpoint, ak, sk, directives='info', directory='logs',
                                file_name_prefix='app.log')
+        self._test_list_objects(bucket, tos_client)
 
+    def test_write_read_object(self):
+        region = os.getenv('TOS_REGION')
+        endpoint = os.getenv('TOS_ENDPOINT')
+        ak = os.getenv('TOS_ACCESS_KEY')
+        sk = os.getenv('TOS_SECRET_KEY')
+        bucket = 'tos-pytorch-connector'
+        tos_client = TosClient(region, endpoint, ak, sk, directives='info', directory='logs',
+                               file_name_prefix='app.log')
+
+        self._test_write_read_object(bucket, tos_client)
+
+    def _test_list_objects(self, bucket, tos_client):
         list_stream = tos_client.list_objects(bucket, '', max_keys=1000)
         count = 0
         try:
@@ -30,15 +59,7 @@ class TestTosClient(unittest.TestCase):
         except TosException as e:
             print(e.args[0].message)
 
-    def test_write_read_object(self):
-        region = os.getenv('TOS_REGION')
-        endpoint = os.getenv('TOS_ENDPOINT')
-        ak = os.getenv('TOS_ACCESS_KEY')
-        sk = os.getenv('TOS_SECRET_KEY')
-        bucket = 'tos-pytorch-connector'
-        tos_client = TosClient(region, endpoint, ak, sk, directives='info', directory='logs',
-                               file_name_prefix='app.log')
-
+    def _test_write_read_object(self, bucket, tos_client):
         key = str(uuid.uuid4())
         read_stream = tos_client.get_object(bucket, key, '', 1)
 
@@ -59,6 +80,8 @@ class TestTosClient(unittest.TestCase):
         write_stream.close()
 
         output = tos_client.head_object(bucket, key)
+        print(output.etag, output.size)
+
         read_stream = tos_client.get_object(bucket, key, output.etag, output.size)
         try:
             offset = 0

{tostorchconnector-1.0.6 → tostorchconnector-1.0.7}/tests/test_tosrawclient.py

@@ -1,3 +1,4 @@
+import io
 import os
 import unittest
 import uuid
@@ -47,6 +48,20 @@ class TestTosRawClient(unittest.TestCase):
         rdata = goutput.read_all()
         assert rdata == data
 
+        goutput: GetObjectOutput = tos_raw_client.get_object(GetObjectInput(bucket, key))
+        assert goutput.status_code == 200
+        assert goutput.etag == poutput.etag
+
+        rdata = io.BytesIO()
+        while 1:
+            chunk = goutput.read()
+            if not chunk:
+                break
+            rdata.write(chunk)
+
+        rdata.seek(0)
+        assert rdata.read() == data
+
         doutput = tos_raw_client.delete_object(DeleteObjectInput(bucket, key))
         assert doutput.status_code == 204
 

{tostorchconnector-1.0.6 → tostorchconnector-1.0.7/tostorchconnector.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: tostorchconnector
-Version: 1.0.6
+Version: 1.0.7
 Summary: TOS connector integration for PyTorch
 Author-email: xiangshijian <xiangshijian@bytedance.com>
 Classifier: Development Status :: 4 - Beta
@@ -19,7 +19,7 @@ Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: torch>=2.0
 Requires-Dist: tos>=2.8.0
-Requires-Dist: tosnativeclient>=1.0.2
+Requires-Dist: tosnativeclient>=1.0.4
 Dynamic: license-file
 
 # TOS Connector for pytorch

tostorchconnector-1.0.7/tostorchconnector.egg-info/requires.txt

@@ -0,0 +1,3 @@
+torch>=2.0
+tos>=2.8.0
+tosnativeclient>=1.0.4

tostorchconnector-1.0.6/tostorchconnector.egg-info/requires.txt

@@ -1,3 +0,0 @@
-torch>=2.0
-tos>=2.8.0
-tosnativeclient>=1.0.2