current-data-py 0.1.5__tar.gz → 0.1.7__tar.gz

{current_data_py-0.1.5/current_data_py.egg-info → current_data_py-0.1.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: current-data-py
-Version: 0.1.5
+Version: 0.1.7
 Summary: Current Data protobuf messages and schema metadata
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown
@@ -13,21 +13,29 @@ Requires-Dist: alibabacloud-oss-v2<2,>=1.4.0
 
 ## Management
 
-`ManagementClient()` connects to `data.current-robotics.work:443` over TLS.
-Pass `address="host:443"` to override it, or supply your own gRPC channel.
+`ManagementClient` connects over TLS. Pass `address=` to configure the service,
+or supply your own gRPC channel.
+
+The examples below read deployment-specific values from environment variables.
+Set `MANAGEMENT_ADDRESS` to your service address and `UPLOAD_DATA_TYPE` to a valid
+integer enum value supplied by your deployment administrator. Set the bucket and
+object-key variables to resources you are authorized to access. These variables
+are read explicitly by the examples; they are not automatic SDK configuration.
 
 ```python
+import os
+
 from current_data_py.management import ManagementClient, Metadata, MetadataFilter
 
-with ManagementClient() as client:
-    upload = client.get_upload_credentials(data_type="CURRENT_V2")
-    read = client.get_read_credentials(bucket="data-infra-raw-data")
+with ManagementClient(address=os.environ["MANAGEMENT_ADDRESS"]) as client:
+    upload = client.get_upload_credentials(data_type=int(os.environ["UPLOAD_DATA_TYPE"]))
+    read = client.get_read_credentials(bucket=os.environ["READ_BUCKET"])
 
     # Register an already uploaded file; replace the example file details.
     record = client.register_data(Metadata(
         name="recording",
-        data_type="DATA_TYPE_CURRENT_V2",
-        storages=[{"bucket": "data-infra-202607", "object_key": "raw_converted/recording.td"}],
+        data_type=int(os.environ["UPLOAD_DATA_TYPE"]),
+        storages=[{"bucket": os.environ["UPLOADED_BUCKET"], "object_key": os.environ["UPLOADED_OBJECT_KEY"]}],
         file_info={"size_bytes": 1234, "checksum": "sha256:<file-hash>"},
     ))
     for item in client.stream_data_references(MetadataFilter(name_prefix="recording")):
@@ -39,25 +47,55 @@ SDK. Registration only saves metadata; it does not upload files. Credential
 calls return fresh credentials without caching. Calls accept `timeout=` and
 `metadata=` for gRPC headers.
 
+## Remote TD reads
+
+`DataClient.open_td()` accepts an optional `bucket` argument. Without it, reads
+use the management service's `OSS_READ_BUCKET` and `OSS_READ_PREFIX` defaults.
+With it, the SDK requests whole-bucket read credentials, limited by the server's
+RAM role, and selects the first matching storage in the reference. An explicit
+bucket does not inherit the default prefix. Default and per-bucket credentials
+are cached separately and refreshed automatically.
+
+```python
+import os
+
+from current_data_py.remote import DataClient
+
+with DataClient(address=os.environ["MANAGEMENT_ADDRESS"]) as client:
+    for reference in client.iter_data():
+        # Use the service's default bucket and prefix.
+        with client.open_td(reference) as reader:
+            print(reader.summary())
+
+        # Request read access to the selected bucket.
+        with client.open_td(reference, bucket=os.environ["READ_BUCKET"]) as reader:
+            print(reader.summary())
+```
+
+The explicit bucket must contain a storage listed in the reference. This uses
+the existing bucket-aware management API; no protocol changes are required.
+
 ## OSS
 
 ```python
+import os
+
 from current_data_py.management import ManagementClient
 from current_data_py.oss import OSSClient
 
-with ManagementClient() as management:
-    with OSSClient(management, permission="read", bucket="data-infra-raw-data") as source:
-        source.download_file("collections/recording/input.h5", "/tmp/input.h5")
+with ManagementClient(address=os.environ["MANAGEMENT_ADDRESS"]) as management:
+    with OSSClient(management, permission="read", bucket=os.environ["READ_BUCKET"]) as source:
+        source.download_file(os.environ["SOURCE_OBJECT_KEY"], "/tmp/input.h5")
 
     # Convert the input into /tmp/output.td before uploading.
-    with OSSClient(management, permission="upload", data_type="CURRENT_V2") as destination:
+    with OSSClient(management, permission="upload", data_type=int(os.environ["UPLOAD_DATA_TYPE"])) as destination:
         key = destination.object_prefix + "recording/output.td"
         destination.upload_file(key, "/tmp/output.td")
         print(destination.bucket, key)  # Use these values when registering metadata.
 ```
 
 - Read access covers the requested bucket, subject to server permissions.
-  Uploads default to `data_type="RAW"`; the server chooses the bucket and prefix.
+  The server chooses the upload bucket and prefix based on the requested data type.
 - Credentials refresh automatically. Expired tokens trigger one refresh and retry.
 - Uploads reject overwrites by default; set `forbid_overwrite=False` to replace.
   Downloads replace the local file only on success; its parent directory must exist.

current_data_py-0.1.7/README.md

@@ -0,0 +1,125 @@
+# current-data-py
+
+## Management
+
+`ManagementClient` connects over TLS. Pass `address=` to configure the service,
+or supply your own gRPC channel.
+
+The examples below read deployment-specific values from environment variables.
+Set `MANAGEMENT_ADDRESS` to your service address and `UPLOAD_DATA_TYPE` to a valid
+integer enum value supplied by your deployment administrator. Set the bucket and
+object-key variables to resources you are authorized to access. These variables
+are read explicitly by the examples; they are not automatic SDK configuration.
+
+```python
+import os
+
+from current_data_py.management import ManagementClient, Metadata, MetadataFilter
+
+with ManagementClient(address=os.environ["MANAGEMENT_ADDRESS"]) as client:
+    upload = client.get_upload_credentials(data_type=int(os.environ["UPLOAD_DATA_TYPE"]))
+    read = client.get_read_credentials(bucket=os.environ["READ_BUCKET"])
+
+    # Register an already uploaded file; replace the example file details.
+    record = client.register_data(Metadata(
+        name="recording",
+        data_type=int(os.environ["UPLOAD_DATA_TYPE"]),
+        storages=[{"bucket": os.environ["UPLOADED_BUCKET"], "object_key": os.environ["UPLOADED_OBJECT_KEY"]}],
+        file_info={"size_bytes": 1234, "checksum": "sha256:<file-hash>"},
+    ))
+    for item in client.stream_data_references(MetadataFilter(name_prefix="recording")):
+        print(item.id, item.storages)
+```
+
+`Metadata`, `MetadataFilter`, and responses are Protobuf objects exported by the
+SDK. Registration only saves metadata; it does not upload files. Credential
+calls return fresh credentials without caching. Calls accept `timeout=` and
+`metadata=` for gRPC headers.
+
+## Remote TD reads
+
+`DataClient.open_td()` accepts an optional `bucket` argument. Without it, reads
+use the management service's `OSS_READ_BUCKET` and `OSS_READ_PREFIX` defaults.
+With it, the SDK requests whole-bucket read credentials, limited by the server's
+RAM role, and selects the first matching storage in the reference. An explicit
+bucket does not inherit the default prefix. Default and per-bucket credentials
+are cached separately and refreshed automatically.
+
+```python
+import os
+
+from current_data_py.remote import DataClient
+
+with DataClient(address=os.environ["MANAGEMENT_ADDRESS"]) as client:
+    for reference in client.iter_data():
+        # Use the service's default bucket and prefix.
+        with client.open_td(reference) as reader:
+            print(reader.summary())
+
+        # Request read access to the selected bucket.
+        with client.open_td(reference, bucket=os.environ["READ_BUCKET"]) as reader:
+            print(reader.summary())
+```
+
+The explicit bucket must contain a storage listed in the reference. This uses
+the existing bucket-aware management API; no protocol changes are required.
+
+## OSS
+
+```python
+import os
+
+from current_data_py.management import ManagementClient
+from current_data_py.oss import OSSClient
+
+with ManagementClient(address=os.environ["MANAGEMENT_ADDRESS"]) as management:
+    with OSSClient(management, permission="read", bucket=os.environ["READ_BUCKET"]) as source:
+        source.download_file(os.environ["SOURCE_OBJECT_KEY"], "/tmp/input.h5")
+
+    # Convert the input into /tmp/output.td before uploading.
+    with OSSClient(management, permission="upload", data_type=int(os.environ["UPLOAD_DATA_TYPE"])) as destination:
+        key = destination.object_prefix + "recording/output.td"
+        destination.upload_file(key, "/tmp/output.td")
+        print(destination.bucket, key)  # Use these values when registering metadata.
+```
+
+- Read access covers the requested bucket, subject to server permissions.
+  The server chooses the upload bucket and prefix based on the requested data type.
+- Credentials refresh automatically. Expired tokens trigger one refresh and retry.
+- Uploads reject overwrites by default; set `forbid_overwrite=False` to replace.
+  Downloads replace the local file only on success; its parent directory must exist.
+- Uploads use a single PutObject request ([maximum 5 GiB](https://www.alibabacloud.com/help/en/oss/developer-reference/upload-files-using-oss-sdk-for-python-v2/)); multipart upload is not supported.
+- Supply `region=` for custom upload endpoints. Deploy the current management
+  credential API before using the new bucket and data-type arguments.
+
+## 打包
+
+```bash
+cd current_data_py  # 从仓库根目录执行
+# 发布新版本前，更新 pyproject.toml 中的 version
+python -m pip install build twine
+rm -rf build dist
+python -m build
+```
+
+## 测试
+
+```bash
+python -m twine check dist/*
+python tests/check_distribution.py
+python -m pip install --force-reinstall dist/*.whl
+python -I tests/check_package.py
+python -I tests/check_management.py
+python -I tests/check_remote.py
+python -I tests/check_oss.py
+```
+
+## 推送
+
+```bash
+# TestPyPI
+python -m twine upload --repository testpypi dist/*
+
+# PyPI
+python -m twine upload dist/*
+```

{current_data_py-0.1.5 → current_data_py-0.1.7/current_data_py.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: current-data-py
-Version: 0.1.5
+Version: 0.1.7
 Summary: Current Data protobuf messages and schema metadata
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown
@@ -13,21 +13,29 @@ Requires-Dist: alibabacloud-oss-v2<2,>=1.4.0
 
 ## Management
 
-`ManagementClient()` connects to `data.current-robotics.work:443` over TLS.
-Pass `address="host:443"` to override it, or supply your own gRPC channel.
+`ManagementClient` connects over TLS. Pass `address=` to configure the service,
+or supply your own gRPC channel.
+
+The examples below read deployment-specific values from environment variables.
+Set `MANAGEMENT_ADDRESS` to your service address and `UPLOAD_DATA_TYPE` to a valid
+integer enum value supplied by your deployment administrator. Set the bucket and
+object-key variables to resources you are authorized to access. These variables
+are read explicitly by the examples; they are not automatic SDK configuration.
 
 ```python
+import os
+
 from current_data_py.management import ManagementClient, Metadata, MetadataFilter
 
-with ManagementClient() as client:
-    upload = client.get_upload_credentials(data_type="CURRENT_V2")
-    read = client.get_read_credentials(bucket="data-infra-raw-data")
+with ManagementClient(address=os.environ["MANAGEMENT_ADDRESS"]) as client:
+    upload = client.get_upload_credentials(data_type=int(os.environ["UPLOAD_DATA_TYPE"]))
+    read = client.get_read_credentials(bucket=os.environ["READ_BUCKET"])
 
     # Register an already uploaded file; replace the example file details.
     record = client.register_data(Metadata(
         name="recording",
-        data_type="DATA_TYPE_CURRENT_V2",
-        storages=[{"bucket": "data-infra-202607", "object_key": "raw_converted/recording.td"}],
+        data_type=int(os.environ["UPLOAD_DATA_TYPE"]),
+        storages=[{"bucket": os.environ["UPLOADED_BUCKET"], "object_key": os.environ["UPLOADED_OBJECT_KEY"]}],
         file_info={"size_bytes": 1234, "checksum": "sha256:<file-hash>"},
     ))
     for item in client.stream_data_references(MetadataFilter(name_prefix="recording")):
@@ -39,25 +47,55 @@ SDK. Registration only saves metadata; it does not upload files. Credential
 calls return fresh credentials without caching. Calls accept `timeout=` and
 `metadata=` for gRPC headers.
 
+## Remote TD reads
+
+`DataClient.open_td()` accepts an optional `bucket` argument. Without it, reads
+use the management service's `OSS_READ_BUCKET` and `OSS_READ_PREFIX` defaults.
+With it, the SDK requests whole-bucket read credentials, limited by the server's
+RAM role, and selects the first matching storage in the reference. An explicit
+bucket does not inherit the default prefix. Default and per-bucket credentials
+are cached separately and refreshed automatically.
+
+```python
+import os
+
+from current_data_py.remote import DataClient
+
+with DataClient(address=os.environ["MANAGEMENT_ADDRESS"]) as client:
+    for reference in client.iter_data():
+        # Use the service's default bucket and prefix.
+        with client.open_td(reference) as reader:
+            print(reader.summary())
+
+        # Request read access to the selected bucket.
+        with client.open_td(reference, bucket=os.environ["READ_BUCKET"]) as reader:
+            print(reader.summary())
+```
+
+The explicit bucket must contain a storage listed in the reference. This uses
+the existing bucket-aware management API; no protocol changes are required.
+
 ## OSS
 
 ```python
+import os
+
 from current_data_py.management import ManagementClient
 from current_data_py.oss import OSSClient
 
-with ManagementClient() as management:
-    with OSSClient(management, permission="read", bucket="data-infra-raw-data") as source:
-        source.download_file("collections/recording/input.h5", "/tmp/input.h5")
+with ManagementClient(address=os.environ["MANAGEMENT_ADDRESS"]) as management:
+    with OSSClient(management, permission="read", bucket=os.environ["READ_BUCKET"]) as source:
+        source.download_file(os.environ["SOURCE_OBJECT_KEY"], "/tmp/input.h5")
 
     # Convert the input into /tmp/output.td before uploading.
-    with OSSClient(management, permission="upload", data_type="CURRENT_V2") as destination:
+    with OSSClient(management, permission="upload", data_type=int(os.environ["UPLOAD_DATA_TYPE"])) as destination:
         key = destination.object_prefix + "recording/output.td"
         destination.upload_file(key, "/tmp/output.td")
         print(destination.bucket, key)  # Use these values when registering metadata.
 ```
 
 - Read access covers the requested bucket, subject to server permissions.
-  Uploads default to `data_type="RAW"`; the server chooses the bucket and prefix.
+  The server chooses the upload bucket and prefix based on the requested data type.
 - Credentials refresh automatically. Expired tokens trigger one refresh and retry.
 - Uploads reject overwrites by default; set `forbid_overwrite=False` to replace.
   Downloads replace the local file only on success; its parent directory must exist.

current_data_py-0.1.7/proto_gen/python/management/service/service_pb2.py

@@ -0,0 +1,64 @@
+# -*- coding: utf-8 -*-
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: management/service/service.proto
+"""Generated protocol buffer code."""
+from google.protobuf.internal import builder as _builder
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import descriptor_pool as _descriptor_pool
+from google.protobuf import symbol_database as _symbol_database
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2
+from management.metadata import metadata_pb2 as management_dot_metadata_dot_metadata__pb2
+
+
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n management/service/service.proto\x12\x15management.service.v1\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\"management/metadata/metadata.proto\"R\n\x1bGetUploadCredentialsRequest\x12\x33\n\tdata_type\x18\x01 \x01(\x0e\x32 .management.metadata.v1.DataType\"\xd5\x01\n\x1cGetUploadCredentialsResponse\x12\x15\n\raccess_key_id\x18\x01 \x01(\t\x12\x19\n\x11\x61\x63\x63\x65ss_key_secret\x18\x02 \x01(\t\x12\x16\n\x0esecurity_token\x18\x03 \x01(\t\x12.\n\nexpires_at\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x14\n\x0coss_endpoint\x18\x05 \x01(\t\x12\x0e\n\x06\x62ucket\x18\x06 \x01(\t\x12\x15\n\robject_prefix\x18\x07 \x01(\t\"r\n\x16\x43reateUploadURLRequest\x12\x0f\n\x07\x64\x61ta_id\x18\x01 \x01(\t\x12\x33\n\tdata_type\x18\x02 \x01(\x0e\x32 .management.metadata.v1.DataType\x12\x12\n\nobject_key\x18\x03 \x01(\t\"z\n\x17\x43reateUploadURLResponse\x12\x0b\n\x03url\x18\x01 \x01(\t\x12\x0e\n\x06\x62ucket\x18\x02 \x01(\t\x12\x12\n\nobject_key\x18\x03 \x01(\t\x12.\n\nexpires_at\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\"E\n\x13RegisterDataRequest\x12.\n\x04\x64\x61ta\x18\x01 \x01(\x0b\x32 .management.metadata.v1.Metadata\"F\n\x14RegisterDataResponse\x12.\n\x04\x64\x61ta\x18\x01 \x01(\x0b\x32 .management.metadata.v1.Metadata\"p\n\x0fListDataRequest\x12\x11\n\tpage_size\x18\x01 \x01(\x05\x12\x12\n\npage_token\x18\x02 \x01(\t\x12\x36\n\x06\x66ilter\x18\x03 \x01(\x0b\x32&.management.metadata.v1.MetadataFilter\"[\n\x10ListDataResponse\x12.\n\x04\x64\x61ta\x18\x01 \x03(\x0b\x32 .management.metadata.v1.Metadata\x12\x17\n\x0fnext_page_token\x18\x02 \x01(\t\"!\n\x0eGetDataRequest\x12\x0f\n\x07\x64\x61ta_id\x18\x01 \x01(\t\"A\n\x0fGetDataResponse\x12.\n\x04\x64\x61ta\x18\x01 \x01(\x0b\x32 .management.metadata.v1.Metadata\"+\n\x18\x43reateDownloadURLRequest\x12\x0f\n\x07\x64\x61ta_id\x18\x01 \x01(\t\"X\n\x19\x43reateDownloadURLResponse\x12\x0b\n\x03url\x18\x01 \x01(\t\x12.\n\nexpires_at\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\"K\n\x11SelectDataRequest\x12\x36\n\x06\x66ilter\x18\x01 \x01(\x0b\x32&.management.metadata.v1.MetadataFilter\"&\n\x12SelectDataResponse\x12\x10\n\x08\x64\x61ta_ids\x18\x01 \x03(\t\"U\n\x1bStreamDataReferencesRequest\x12\x36\n\x06\x66ilter\x18\x01 \x01(\x0b\x32&.management.metadata.v1.MetadataFilter\"N\n\x1cStreamDataReferencesResponse\x12.\n\x04\x64\x61ta\x18\x01 \x03(\x0b\x32 .management.metadata.v1.Metadata\"+\n\x19GetReadCredentialsRequest\x12\x0e\n\x06\x62ucket\x18\x01 \x01(\t\"\xcc\x01\n\x1aGetReadCredentialsResponse\x12\x15\n\raccess_key_id\x18\x01 \x01(\t\x12\x19\n\x11\x61\x63\x63\x65ss_key_secret\x18\x02 \x01(\t\x12\x16\n\x0esecurity_token\x18\x03 \x01(\t\x12.\n\nexpires_at\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x14\n\x0coss_endpoint\x18\x05 \x01(\t\x12\x0e\n\x06region\x18\x06 \x01(\t\x12\x0e\n\x06\x62ucket\x18\x07 \x01(\t2\x84\x08\n\x15\x44\x61taManagementService\x12\x81\x01\n\x14StreamDataReferences\x12\x32.management.service.v1.StreamDataReferencesRequest\x1a\x33.management.service.v1.StreamDataReferencesResponse0\x01\x12y\n\x12GetReadCredentials\x12\x30.management.service.v1.GetReadCredentialsRequest\x1a\x31.management.service.v1.GetReadCredentialsResponse\x12g\n\x0cRegisterData\x12*.management.service.v1.RegisterDataRequest\x1a+.management.service.v1.RegisterDataResponse\x12p\n\x0f\x43reateUploadURL\x12-.management.service.v1.CreateUploadURLRequest\x1a..management.service.v1.CreateUploadURLResponse\x12\x7f\n\x14GetUploadCredentials\x12\x32.management.service.v1.GetUploadCredentialsRequest\x1a\x33.management.service.v1.GetUploadCredentialsResponse\x12[\n\x08ListData\x12&.management.service.v1.ListDataRequest\x1a\'.management.service.v1.ListDataResponse\x12X\n\x07GetData\x12%.management.service.v1.GetDataRequest\x1a&.management.service.v1.GetDataResponse\x12v\n\x11\x43reateDownloadURL\x12/.management.service.v1.CreateDownloadURLRequest\x1a\x30.management.service.v1.CreateDownloadURLResponse\x12\x61\n\nSelectData\x12(.management.service.v1.SelectDataRequest\x1a).management.service.v1.SelectDataResponseB(Z&proto_gen/management/service;servicepbb\x06proto3')
+
+_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals())
+_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'management.service.service_pb2', globals())
+if _descriptor._USE_C_DESCRIPTORS == False:
+
+  DESCRIPTOR._options = None
+  DESCRIPTOR._serialized_options = b'Z&proto_gen/management/service;servicepb'
+  _GETUPLOADCREDENTIALSREQUEST._serialized_start=128
+  _GETUPLOADCREDENTIALSREQUEST._serialized_end=210
+  _GETUPLOADCREDENTIALSRESPONSE._serialized_start=213
+  _GETUPLOADCREDENTIALSRESPONSE._serialized_end=426
+  _CREATEUPLOADURLREQUEST._serialized_start=428
+  _CREATEUPLOADURLREQUEST._serialized_end=542
+  _CREATEUPLOADURLRESPONSE._serialized_start=544
+  _CREATEUPLOADURLRESPONSE._serialized_end=666
+  _REGISTERDATAREQUEST._serialized_start=668
+  _REGISTERDATAREQUEST._serialized_end=737
+  _REGISTERDATARESPONSE._serialized_start=739
+  _REGISTERDATARESPONSE._serialized_end=809
+  _LISTDATAREQUEST._serialized_start=811
+  _LISTDATAREQUEST._serialized_end=923
+  _LISTDATARESPONSE._serialized_start=925
+  _LISTDATARESPONSE._serialized_end=1016
+  _GETDATAREQUEST._serialized_start=1018
+  _GETDATAREQUEST._serialized_end=1051
+  _GETDATARESPONSE._serialized_start=1053
+  _GETDATARESPONSE._serialized_end=1118
+  _CREATEDOWNLOADURLREQUEST._serialized_start=1120
+  _CREATEDOWNLOADURLREQUEST._serialized_end=1163
+  _CREATEDOWNLOADURLRESPONSE._serialized_start=1165
+  _CREATEDOWNLOADURLRESPONSE._serialized_end=1253
+  _SELECTDATAREQUEST._serialized_start=1255
+  _SELECTDATAREQUEST._serialized_end=1330
+  _SELECTDATARESPONSE._serialized_start=1332
+  _SELECTDATARESPONSE._serialized_end=1370
+  _STREAMDATAREFERENCESREQUEST._serialized_start=1372
+  _STREAMDATAREFERENCESREQUEST._serialized_end=1457
+  _STREAMDATAREFERENCESRESPONSE._serialized_start=1459
+  _STREAMDATAREFERENCESRESPONSE._serialized_end=1537
+  _GETREADCREDENTIALSREQUEST._serialized_start=1539
+  _GETREADCREDENTIALSREQUEST._serialized_end=1582
+  _GETREADCREDENTIALSRESPONSE._serialized_start=1585
+  _GETREADCREDENTIALSRESPONSE._serialized_end=1789
+  _DATAMANAGEMENTSERVICE._serialized_start=1792
+  _DATAMANAGEMENTSERVICE._serialized_end=2820
+# @@protoc_insertion_point(module_scope)

current_data_py-0.1.7/proto_gen/python/msg/custom_msgs/pose_rpy_pb2.py

@@ -0,0 +1,26 @@
+# -*- coding: utf-8 -*-
+# Generated by the protocol buffer compiler.  DO NOT EDIT!
+# source: custom_msgs/pose_rpy.proto
+"""Generated protocol buffer code."""
+from google.protobuf.internal import builder as _builder
+from google.protobuf import descriptor as _descriptor
+from google.protobuf import descriptor_pool as _descriptor_pool
+from google.protobuf import symbol_database as _symbol_database
+# @@protoc_insertion_point(imports)
+
+_sym_db = _symbol_database.Default()
+
+
+from geometry_msgs import point_pb2 as geometry__msgs_dot_point__pb2
+
+
+DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1a\x63ustom_msgs/pose_rpy.proto\x12\x0b\x63ustom_msgs\x1a\x19geometry_msgs/point.proto\"[\n\x07PoseRPY\x12&\n\x08position\x18\x01 \x01(\x0b\x32\x14.geometry_msgs.Point\x12\x0c\n\x04roll\x18\x02 \x01(\x01\x12\r\n\x05pitch\x18\x03 \x01(\x01\x12\x0b\n\x03yaw\x18\x04 \x01(\x01\x62\x06proto3')
+
+_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals())
+_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'custom_msgs.pose_rpy_pb2', globals())
+if _descriptor._USE_C_DESCRIPTORS == False:
+
+  DESCRIPTOR._options = None
+  _POSERPY._serialized_start=70
+  _POSERPY._serialized_end=161
+# @@protoc_insertion_point(module_scope)

{current_data_py-0.1.5 → current_data_py-0.1.7}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "current-data-py"
-version = "0.1.5"
+version = "0.1.7"
 description = "Current Data protobuf messages and schema metadata"
 readme = "README.md"
 requires-python = ">=3.9"

{current_data_py-0.1.5 → current_data_py-0.1.7}/remote/client.py

@@ -13,8 +13,9 @@ from .source import OSSReadSource
 
 
 class _Credentials(oss.credentials.CredentialsProvider):
-    def __init__(self, client):
+    def __init__(self, client, bucket=None):
         self.client = client
+        self.bucket = bucket
         self.lock = threading.Lock()
         self.cached = None
         self.scope = None
@@ -22,7 +23,10 @@ class _Credentials(oss.credentials.CredentialsProvider):
     def get(self):
         with self.lock:
             if self.cached is None or self.cached.expires_at.seconds <= time.time() + 300:
-                value = self.client.get_read_credentials()
+                value = (self.client.get_read_credentials() if self.bucket is None
+                         else self.client.get_read_credentials(bucket=self.bucket))
+                if self.bucket is not None and value.bucket != self.bucket:
+                    raise ValueError("Management returned a different read bucket")
                 scope = (value.oss_endpoint, value.region, value.bucket)
                 if (not all(scope) or not all((value.access_key_id, value.access_key_secret, value.security_token))
                         or value.expires_at.seconds <= time.time() + 300):
@@ -48,14 +52,32 @@ class DataClient(ManagementClient):
     def __init__(self, channel=None, *, address=None):
         super().__init__(channel, address=address)
         self._credentials = _Credentials(self)
+        # Default scope may restrict a prefix even when it names the same bucket.
+        self._credentials_by_bucket = {None: self._credentials}
+        self._credentials_lock = threading.Lock()
 
     def iter_data(self, filter=None, *, timeout=None):
         """Yield partial Metadata (ID, storages and file size). Close to cancel early."""
         return self.stream_data_references(filter, timeout=timeout)
 
     @contextmanager
-    def open_td(self, reference):
-        """Read the first storage in the credential bucket; keep open until reads finish."""
-        with OSSReadSource(reference, self._credentials) as source:
+    def open_td(self, reference, *, bucket=None):
+        """Read TD from the first storage matching the credential bucket.
+
+        Supply bucket to request whole-bucket reads, subject to the server's RAM
+        role. Omit it (or pass None) to use the management service's configured
+        default bucket and prefix. Keep the context open until reads finish.
+        """
+        if bucket is not None and (
+            not isinstance(bucket, str) or not bucket or bucket != bucket.strip()
+            or any(c in bucket for c in "*?/:\\")
+        ):
+            raise ValueError("bucket must be a nonempty bucket name")
+        with self._credentials_lock:
+            credentials = self._credentials_by_bucket.get(bucket)
+            if credentials is None:
+                credentials = _Credentials(self, bucket)
+                self._credentials_by_bucket[bucket] = credentials
+        with OSSReadSource(reference, credentials) as source:
             yield Reader(source)
 

{current_data_py-0.1.5 → current_data_py-0.1.7}/schema/__init__.py

@@ -9,6 +9,7 @@ from .._proto.foxglove.compressed_video_pb2 import CompressedVideo
 from .._proto.geometry_msgs.point_pb2 import Point
 from .._proto.geometry_msgs.point32_pb2 import Point32
 from .._proto.geometry_msgs.pose_pb2 import Pose
+from .._proto.custom_msgs.pose_rpy_pb2 import PoseRPY
 from .._proto.geometry_msgs.pose_stamped_pb2 import PoseStamped
 from .._proto.geometry_msgs.pose_with_covariance_pb2 import PoseWithCovariance
 from .._proto.geometry_msgs.pose_with_covariance_stamped_pb2 import PoseWithCovarianceStamped
@@ -62,6 +63,7 @@ SCHEMA_TO_CLASS = {
     "geometry_msgs.Point": Point,
     "geometry_msgs.Point32": Point32,
     "geometry_msgs.Pose": Pose,
+    "custom_msgs.PoseRPY": PoseRPY,
     "geometry_msgs.PoseStamped": PoseStamped,
     "geometry_msgs.PoseWithCovariance": PoseWithCovariance,
     "geometry_msgs.PoseWithCovarianceStamped": PoseWithCovarianceStamped,

{current_data_py-0.1.5 → current_data_py-0.1.7}/tests/check_remote.py

@@ -114,13 +114,14 @@ class RemoteTest(unittest.TestCase):
                 writer.write_message("/pose", msg.SerializeToString(), timestamp)
             writer.close_topic()
         data = output.getvalue()
-        state = {"mode": "ok", "gets": 0, "credentials": 0, "cancelled": threading.Event()}
+        state = {"mode": "ok", "gets": 0, "credentials": 0, "credential_buckets": [], "oss_requests": [], "cancelled": threading.Event()}
 
         class Handler(BaseHTTPRequestHandler):
             def log_message(self, *args):
                 pass
 
             def do_HEAD(self):
+                state["oss_requests"].append((self.path, self.headers.get("x-oss-security-token")))
                 self.send_response(200)
                 self.send_header("Content-Length", str(len(data)))
                 self.send_header("ETag", '"etag"')
@@ -152,9 +153,10 @@ class RemoteTest(unittest.TestCase):
         class Management(rpc.DataManagementServiceServicer):
             def GetReadCredentials(self, request, context):
                 state["credentials"] += 1
+                state["credential_buckets"].append(request.bucket)
                 value = pb.GetReadCredentialsResponse(access_key_id="id", access_key_secret="secret",
                     security_token=f"token-{state['credentials']}", oss_endpoint=f"http://127.0.0.1:{http.server_port}",
-                    region="cn-hangzhou", bucket="bucket")
+                    region="cn-hangzhou", bucket=request.bucket or "bucket")
                 value.expires_at.seconds = int(time.time()) + 3600
                 return value
 
@@ -225,6 +227,62 @@ class RemoteTest(unittest.TestCase):
             self.assertEqual([line.split()[:3] for line in result.stdout.splitlines()],
                              [["id", str(t), "/pose"] for t in (3, 4, 5)])
 
+            # Default scope and explicit reads of the same bucket cannot share
+            # credentials: only the latter requests whole-bucket permission.
+            multi = Metadata(storages=[Storage(bucket="bucket", object_key="outside/file.td"),
+                                       Storage(bucket="other", object_key="nested/file.td")])
+            def read(bucket):
+                with client.open_td(multi, bucket=bucket) as reader:
+                    self.assertEqual(len(list(reader.read_messages())), 10)
+                return state["oss_requests"][-1]
+
+            baseline = len(state["credential_buckets"])
+            first = read("bucket")
+            second = read("other")
+            self.assertEqual(state["credential_buckets"][baseline:], ["bucket", "other"])
+            self.assertEqual(first[0], "/bucket/outside/file.td")
+            self.assertEqual(second[0], "/other/nested/file.td")
+            self.assertNotEqual(first[1], second[1])
+            self.assertNotEqual(first[1], client._credentials.cached.security_token)
+            self.assertEqual(read("bucket"), first)
+            self.assertEqual(read("other"), second)
+            self.assertEqual(len(state["credential_buckets"]), baseline + 2)
+            with client.open_td(reference):
+                pass
+            self.assertEqual(len(state["credential_buckets"]), baseline + 2)
+
+            # Expiry recovery stays on the explicitly requested bucket.
+            state["mode"], state["old_token"] = "expired", first[1]
+            read("bucket")
+            self.assertEqual(state["credential_buckets"][baseline:], ["bucket", "other", "bucket"])
+            state["mode"] = "ok"
+            self.assertEqual(read("other"), second)
+            before = len(state["credential_buckets"])
+            with patch("current_data_py.remote.client.time.time", return_value=time.time()+3400):
+                with ThreadPoolExecutor(max_workers=4) as pool:
+                    list(pool.map(read, ["other"] * 4))
+            self.assertEqual(state["credential_buckets"][before:], ["other"])
+
+    def test_invalid_bucket_fails_before_credentials(self):
+        with DataClient(MagicMock()) as client, patch.object(client, "get_read_credentials") as fetch:
+            for bucket in ("", " ", " bucket", "bucket ", "bucket/path", "oss://bucket", "*", "a?b", 123):
+                with self.subTest(bucket=bucket), self.assertRaises(ValueError):
+                    with client.open_td(Metadata(), bucket=bucket):
+                        self.fail("invalid bucket accepted")
+            fetch.assert_not_called()
+
+    def test_explicit_bucket_mismatch_does_not_read_objects(self):
+        with DataClient(MagicMock()) as client, patch.object(client, "get_read_credentials") as fetch, \
+                patch.object(OSSReadSource, "_request") as request:
+            fetch.return_value = pb.GetReadCredentialsResponse(bucket="wrong-bucket")
+            reference = Metadata(storages=[Storage(bucket="wrong-bucket", object_key="file.td")])
+            with self.assertRaisesRegex(ValueError, "different read bucket"):
+                with client.open_td(reference, bucket="requested-bucket"):
+                    self.fail("mismatched credentials accepted")
+            fetch.assert_called_once_with(bucket="requested-bucket")
+            request.assert_not_called()
+
+
 
 
 if __name__ == "__main__":

current_data_py-0.1.5/README.md

@@ -1,87 +0,0 @@
-# current-data-py
-
-## Management
-
-`ManagementClient()` connects to `data.current-robotics.work:443` over TLS.
-Pass `address="host:443"` to override it, or supply your own gRPC channel.
-
-```python
-from current_data_py.management import ManagementClient, Metadata, MetadataFilter
-
-with ManagementClient() as client:
-    upload = client.get_upload_credentials(data_type="CURRENT_V2")
-    read = client.get_read_credentials(bucket="data-infra-raw-data")
-
-    # Register an already uploaded file; replace the example file details.
-    record = client.register_data(Metadata(
-        name="recording",
-        data_type="DATA_TYPE_CURRENT_V2",
-        storages=[{"bucket": "data-infra-202607", "object_key": "raw_converted/recording.td"}],
-        file_info={"size_bytes": 1234, "checksum": "sha256:<file-hash>"},
-    ))
-    for item in client.stream_data_references(MetadataFilter(name_prefix="recording")):
-        print(item.id, item.storages)
-```
-
-`Metadata`, `MetadataFilter`, and responses are Protobuf objects exported by the
-SDK. Registration only saves metadata; it does not upload files. Credential
-calls return fresh credentials without caching. Calls accept `timeout=` and
-`metadata=` for gRPC headers.
-
-## OSS
-
-```python
-from current_data_py.management import ManagementClient
-from current_data_py.oss import OSSClient
-
-with ManagementClient() as management:
-    with OSSClient(management, permission="read", bucket="data-infra-raw-data") as source:
-        source.download_file("collections/recording/input.h5", "/tmp/input.h5")
-
-    # Convert the input into /tmp/output.td before uploading.
-    with OSSClient(management, permission="upload", data_type="CURRENT_V2") as destination:
-        key = destination.object_prefix + "recording/output.td"
-        destination.upload_file(key, "/tmp/output.td")
-        print(destination.bucket, key)  # Use these values when registering metadata.
-```
-
-- Read access covers the requested bucket, subject to server permissions.
-  Uploads default to `data_type="RAW"`; the server chooses the bucket and prefix.
-- Credentials refresh automatically. Expired tokens trigger one refresh and retry.
-- Uploads reject overwrites by default; set `forbid_overwrite=False` to replace.
-  Downloads replace the local file only on success; its parent directory must exist.
-- Uploads use a single PutObject request ([maximum 5 GiB](https://www.alibabacloud.com/help/en/oss/developer-reference/upload-files-using-oss-sdk-for-python-v2/)); multipart upload is not supported.
-- Supply `region=` for custom upload endpoints. Deploy the current management
-  credential API before using the new bucket and data-type arguments.
-
-## 打包
-
-```bash
-cd current_data_py  # 从仓库根目录执行
-# 发布新版本前，更新 pyproject.toml 中的 version
-python -m pip install build twine
-rm -rf build dist
-python -m build
-```
-
-## 测试
-
-```bash
-python -m twine check dist/*
-python tests/check_distribution.py
-python -m pip install --force-reinstall dist/*.whl
-python -I tests/check_package.py
-python -I tests/check_management.py
-python -I tests/check_remote.py
-python -I tests/check_oss.py
-```
-
-## 推送
-
-```bash
-# TestPyPI
-python -m twine upload --repository testpypi dist/*
-
-# PyPI
-python -m twine upload dist/*
-```

current_data_py-0.1.5/proto_gen/python/management/service/service_pb2.py

@@ -1,64 +0,0 @@
-# -*- coding: utf-8 -*-
-# Generated by the protocol buffer compiler.  DO NOT EDIT!
-# source: management/service/service.proto
-"""Generated protocol buffer code."""
-from google.protobuf.internal import builder as _builder
-from google.protobuf import descriptor as _descriptor
-from google.protobuf import descriptor_pool as _descriptor_pool
-from google.protobuf import symbol_database as _symbol_database
-# @@protoc_insertion_point(imports)
-
-_sym_db = _symbol_database.Default()
-
-
-from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2
-from management.metadata import metadata_pb2 as management_dot_metadata_dot_metadata__pb2
-
-
-DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n management/service/service.proto\x12\x15management.service.v1\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\"management/metadata/metadata.proto\"R\n\x1bGetUploadCredentialsRequest\x12\x33\n\tdata_type\x18\x01 \x01(\x0e\x32 .management.metadata.v1.DataType\"\xd5\x01\n\x1cGetUploadCredentialsResponse\x12\x15\n\raccess_key_id\x18\x01 \x01(\t\x12\x19\n\x11\x61\x63\x63\x65ss_key_secret\x18\x02 \x01(\t\x12\x16\n\x0esecurity_token\x18\x03 \x01(\t\x12.\n\nexpires_at\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x14\n\x0coss_endpoint\x18\x05 \x01(\t\x12\x0e\n\x06\x62ucket\x18\x06 \x01(\t\x12\x15\n\robject_prefix\x18\x07 \x01(\t\")\n\x16\x43reateUploadURLRequest\x12\x0f\n\x07\x64\x61ta_id\x18\x01 \x01(\t\"z\n\x17\x43reateUploadURLResponse\x12\x0b\n\x03url\x18\x01 \x01(\t\x12\x0e\n\x06\x62ucket\x18\x02 \x01(\t\x12\x12\n\nobject_key\x18\x03 \x01(\t\x12.\n\nexpires_at\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\"E\n\x13RegisterDataRequest\x12.\n\x04\x64\x61ta\x18\x01 \x01(\x0b\x32 .management.metadata.v1.Metadata\"F\n\x14RegisterDataResponse\x12.\n\x04\x64\x61ta\x18\x01 \x01(\x0b\x32 .management.metadata.v1.Metadata\"p\n\x0fListDataRequest\x12\x11\n\tpage_size\x18\x01 \x01(\x05\x12\x12\n\npage_token\x18\x02 \x01(\t\x12\x36\n\x06\x66ilter\x18\x03 \x01(\x0b\x32&.management.metadata.v1.MetadataFilter\"[\n\x10ListDataResponse\x12.\n\x04\x64\x61ta\x18\x01 \x03(\x0b\x32 .management.metadata.v1.Metadata\x12\x17\n\x0fnext_page_token\x18\x02 \x01(\t\"!\n\x0eGetDataRequest\x12\x0f\n\x07\x64\x61ta_id\x18\x01 \x01(\t\"A\n\x0fGetDataResponse\x12.\n\x04\x64\x61ta\x18\x01 \x01(\x0b\x32 .management.metadata.v1.Metadata\"+\n\x18\x43reateDownloadURLRequest\x12\x0f\n\x07\x64\x61ta_id\x18\x01 \x01(\t\"X\n\x19\x43reateDownloadURLResponse\x12\x0b\n\x03url\x18\x01 \x01(\t\x12.\n\nexpires_at\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\"K\n\x11SelectDataRequest\x12\x36\n\x06\x66ilter\x18\x01 \x01(\x0b\x32&.management.metadata.v1.MetadataFilter\"&\n\x12SelectDataResponse\x12\x10\n\x08\x64\x61ta_ids\x18\x01 \x03(\t\"U\n\x1bStreamDataReferencesRequest\x12\x36\n\x06\x66ilter\x18\x01 \x01(\x0b\x32&.management.metadata.v1.MetadataFilter\"N\n\x1cStreamDataReferencesResponse\x12.\n\x04\x64\x61ta\x18\x01 \x03(\x0b\x32 .management.metadata.v1.Metadata\"+\n\x19GetReadCredentialsRequest\x12\x0e\n\x06\x62ucket\x18\x01 \x01(\t\"\xcc\x01\n\x1aGetReadCredentialsResponse\x12\x15\n\raccess_key_id\x18\x01 \x01(\t\x12\x19\n\x11\x61\x63\x63\x65ss_key_secret\x18\x02 \x01(\t\x12\x16\n\x0esecurity_token\x18\x03 \x01(\t\x12.\n\nexpires_at\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x14\n\x0coss_endpoint\x18\x05 \x01(\t\x12\x0e\n\x06region\x18\x06 \x01(\t\x12\x0e\n\x06\x62ucket\x18\x07 \x01(\t2\x84\x08\n\x15\x44\x61taManagementService\x12\x81\x01\n\x14StreamDataReferences\x12\x32.management.service.v1.StreamDataReferencesRequest\x1a\x33.management.service.v1.StreamDataReferencesResponse0\x01\x12y\n\x12GetReadCredentials\x12\x30.management.service.v1.GetReadCredentialsRequest\x1a\x31.management.service.v1.GetReadCredentialsResponse\x12g\n\x0cRegisterData\x12*.management.service.v1.RegisterDataRequest\x1a+.management.service.v1.RegisterDataResponse\x12p\n\x0f\x43reateUploadURL\x12-.management.service.v1.CreateUploadURLRequest\x1a..management.service.v1.CreateUploadURLResponse\x12\x7f\n\x14GetUploadCredentials\x12\x32.management.service.v1.GetUploadCredentialsRequest\x1a\x33.management.service.v1.GetUploadCredentialsResponse\x12[\n\x08ListData\x12&.management.service.v1.ListDataRequest\x1a\'.management.service.v1.ListDataResponse\x12X\n\x07GetData\x12%.management.service.v1.GetDataRequest\x1a&.management.service.v1.GetDataResponse\x12v\n\x11\x43reateDownloadURL\x12/.management.service.v1.CreateDownloadURLRequest\x1a\x30.management.service.v1.CreateDownloadURLResponse\x12\x61\n\nSelectData\x12(.management.service.v1.SelectDataRequest\x1a).management.service.v1.SelectDataResponseB(Z&proto_gen/management/service;servicepbb\x06proto3')
-
-_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals())
-_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'management.service.service_pb2', globals())
-if _descriptor._USE_C_DESCRIPTORS == False:
-
-  DESCRIPTOR._options = None
-  DESCRIPTOR._serialized_options = b'Z&proto_gen/management/service;servicepb'
-  _GETUPLOADCREDENTIALSREQUEST._serialized_start=128
-  _GETUPLOADCREDENTIALSREQUEST._serialized_end=210
-  _GETUPLOADCREDENTIALSRESPONSE._serialized_start=213
-  _GETUPLOADCREDENTIALSRESPONSE._serialized_end=426
-  _CREATEUPLOADURLREQUEST._serialized_start=428
-  _CREATEUPLOADURLREQUEST._serialized_end=469
-  _CREATEUPLOADURLRESPONSE._serialized_start=471
-  _CREATEUPLOADURLRESPONSE._serialized_end=593
-  _REGISTERDATAREQUEST._serialized_start=595
-  _REGISTERDATAREQUEST._serialized_end=664
-  _REGISTERDATARESPONSE._serialized_start=666
-  _REGISTERDATARESPONSE._serialized_end=736
-  _LISTDATAREQUEST._serialized_start=738
-  _LISTDATAREQUEST._serialized_end=850
-  _LISTDATARESPONSE._serialized_start=852
-  _LISTDATARESPONSE._serialized_end=943
-  _GETDATAREQUEST._serialized_start=945
-  _GETDATAREQUEST._serialized_end=978
-  _GETDATARESPONSE._serialized_start=980
-  _GETDATARESPONSE._serialized_end=1045
-  _CREATEDOWNLOADURLREQUEST._serialized_start=1047
-  _CREATEDOWNLOADURLREQUEST._serialized_end=1090
-  _CREATEDOWNLOADURLRESPONSE._serialized_start=1092
-  _CREATEDOWNLOADURLRESPONSE._serialized_end=1180
-  _SELECTDATAREQUEST._serialized_start=1182
-  _SELECTDATAREQUEST._serialized_end=1257
-  _SELECTDATARESPONSE._serialized_start=1259
-  _SELECTDATARESPONSE._serialized_end=1297
-  _STREAMDATAREFERENCESREQUEST._serialized_start=1299
-  _STREAMDATAREFERENCESREQUEST._serialized_end=1384
-  _STREAMDATAREFERENCESRESPONSE._serialized_start=1386
-  _STREAMDATAREFERENCESRESPONSE._serialized_end=1464
-  _GETREADCREDENTIALSREQUEST._serialized_start=1466
-  _GETREADCREDENTIALSREQUEST._serialized_end=1509
-  _GETREADCREDENTIALSRESPONSE._serialized_start=1512
-  _GETREADCREDENTIALSRESPONSE._serialized_end=1716
-  _DATAMANAGEMENTSERVICE._serialized_start=1719
-  _DATAMANAGEMENTSERVICE._serialized_end=2747
-# @@protoc_insertion_point(module_scope)