current-data-py 0.1.4__tar.gz → 0.1.6__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {current_data_py-0.1.4 → current_data_py-0.1.6}/MANIFEST.in +4 -3
- current_data_py-0.1.6/PKG-INFO +136 -0
- current_data_py-0.1.6/README.md +125 -0
- {current_data_py-0.1.4 → current_data_py-0.1.6}/__init__.py +8 -8
- current_data_py-0.1.6/current_data_py.egg-info/PKG-INFO +136 -0
- {current_data_py-0.1.4 → current_data_py-0.1.6}/current_data_py.egg-info/SOURCES.txt +3 -0
- {current_data_py-0.1.4 → current_data_py-0.1.6}/examples/__init__.py +1 -1
- {current_data_py-0.1.4 → current_data_py-0.1.6}/examples/list_schemas.py +7 -7
- {current_data_py-0.1.4 → current_data_py-0.1.6}/examples/read_remote_td.py +54 -54
- {current_data_py-0.1.4 → current_data_py-0.1.6}/examples/read_td.py +30 -30
- {current_data_py-0.1.4 → current_data_py-0.1.6}/examples/write_td.py +30 -30
- {current_data_py-0.1.4 → current_data_py-0.1.6}/management.py +114 -98
- current_data_py-0.1.6/oss.py +182 -0
- {current_data_py-0.1.4 → current_data_py-0.1.6}/proto_gen/python/management/metadata/metadata_pb2.py +62 -62
- current_data_py-0.1.6/proto_gen/python/management/service/service_pb2.py +64 -0
- {current_data_py-0.1.4 → current_data_py-0.1.6}/proto_gen/python/management/service/service_pb2_grpc.py +330 -330
- {current_data_py-0.1.4 → current_data_py-0.1.6}/proto_gen/python/msg/builtin_interfaces/duration_pb2.py +25 -25
- {current_data_py-0.1.4 → current_data_py-0.1.6}/proto_gen/python/msg/builtin_interfaces/time_pb2.py +25 -25
- {current_data_py-0.1.4 → current_data_py-0.1.6}/proto_gen/python/msg/custom_msgs/joint_current_pb2.py +26 -26
- {current_data_py-0.1.4 → current_data_py-0.1.6}/proto_gen/python/msg/custom_msgs/slam_quality_pb2.py +25 -25
- {current_data_py-0.1.4 → current_data_py-0.1.6}/proto_gen/python/msg/custom_msgs/sonic_frame_pb2.py +25 -25
- {current_data_py-0.1.4 → current_data_py-0.1.6}/proto_gen/python/msg/custom_msgs/target_quality_pb2.py +25 -25
- {current_data_py-0.1.4 → current_data_py-0.1.6}/proto_gen/python/msg/foxglove/compressed_audio_pb2.py +26 -26
- {current_data_py-0.1.4 → current_data_py-0.1.6}/proto_gen/python/msg/foxglove/compressed_video_pb2.py +26 -26
- {current_data_py-0.1.4 → current_data_py-0.1.6}/proto_gen/python/msg/geometry_msgs/point32_pb2.py +25 -25
- {current_data_py-0.1.4 → current_data_py-0.1.6}/proto_gen/python/msg/geometry_msgs/point_pb2.py +25 -25
- {current_data_py-0.1.4 → current_data_py-0.1.6}/proto_gen/python/msg/geometry_msgs/pose_pb2.py +27 -27
- {current_data_py-0.1.4 → current_data_py-0.1.6}/proto_gen/python/msg/geometry_msgs/pose_stamped_pb2.py +27 -27
- {current_data_py-0.1.4 → current_data_py-0.1.6}/proto_gen/python/msg/geometry_msgs/pose_with_covariance_pb2.py +26 -26
- {current_data_py-0.1.4 → current_data_py-0.1.6}/proto_gen/python/msg/geometry_msgs/pose_with_covariance_stamped_pb2.py +27 -27
- {current_data_py-0.1.4 → current_data_py-0.1.6}/proto_gen/python/msg/geometry_msgs/quaternion_pb2.py +25 -25
- {current_data_py-0.1.4 → current_data_py-0.1.6}/proto_gen/python/msg/geometry_msgs/transform_pb2.py +27 -27
- {current_data_py-0.1.4 → current_data_py-0.1.6}/proto_gen/python/msg/geometry_msgs/transform_stamped_pb2.py +27 -27
- {current_data_py-0.1.4 → current_data_py-0.1.6}/proto_gen/python/msg/geometry_msgs/twist_pb2.py +26 -26
- {current_data_py-0.1.4 → current_data_py-0.1.6}/proto_gen/python/msg/geometry_msgs/twist_stamped_pb2.py +27 -27
- {current_data_py-0.1.4 → current_data_py-0.1.6}/proto_gen/python/msg/geometry_msgs/twist_with_covariance_pb2.py +26 -26
- {current_data_py-0.1.4 → current_data_py-0.1.6}/proto_gen/python/msg/geometry_msgs/twist_with_covariance_stamped_pb2.py +27 -27
- {current_data_py-0.1.4 → current_data_py-0.1.6}/proto_gen/python/msg/geometry_msgs/vector3_pb2.py +25 -25
- {current_data_py-0.1.4 → current_data_py-0.1.6}/proto_gen/python/msg/geometry_msgs/wrench_pb2.py +26 -26
- {current_data_py-0.1.4 → current_data_py-0.1.6}/proto_gen/python/msg/geometry_msgs/wrench_stamped_pb2.py +27 -27
- {current_data_py-0.1.4 → current_data_py-0.1.6}/proto_gen/python/msg/nav_msgs/map_meta_data_pb2.py +27 -27
- {current_data_py-0.1.4 → current_data_py-0.1.6}/proto_gen/python/msg/nav_msgs/occupancy_grid_pb2.py +27 -27
- {current_data_py-0.1.4 → current_data_py-0.1.6}/proto_gen/python/msg/nav_msgs/odometry_pb2.py +28 -28
- {current_data_py-0.1.4 → current_data_py-0.1.6}/proto_gen/python/msg/nav_msgs/path_pb2.py +27 -27
- {current_data_py-0.1.4 → current_data_py-0.1.6}/proto_gen/python/msg/sensor_msgs/battery_state_pb2.py +32 -32
- {current_data_py-0.1.4 → current_data_py-0.1.6}/proto_gen/python/msg/sensor_msgs/camera_info_pb2.py +27 -27
- {current_data_py-0.1.4 → current_data_py-0.1.6}/proto_gen/python/msg/sensor_msgs/compressed_image_pb2.py +26 -26
- {current_data_py-0.1.4 → current_data_py-0.1.6}/proto_gen/python/msg/sensor_msgs/image_pb2.py +26 -26
- {current_data_py-0.1.4 → current_data_py-0.1.6}/proto_gen/python/msg/sensor_msgs/imu_pb2.py +28 -28
- {current_data_py-0.1.4 → current_data_py-0.1.6}/proto_gen/python/msg/sensor_msgs/joint_state_pb2.py +26 -26
- {current_data_py-0.1.4 → current_data_py-0.1.6}/proto_gen/python/msg/sensor_msgs/laser_scan_pb2.py +26 -26
- {current_data_py-0.1.4 → current_data_py-0.1.6}/proto_gen/python/msg/sensor_msgs/nav_sat_fix_pb2.py +29 -29
- {current_data_py-0.1.4 → current_data_py-0.1.6}/proto_gen/python/msg/sensor_msgs/nav_sat_status_pb2.py +27 -27
- {current_data_py-0.1.4 → current_data_py-0.1.6}/proto_gen/python/msg/sensor_msgs/point_cloud2_pb2.py +27 -27
- {current_data_py-0.1.4 → current_data_py-0.1.6}/proto_gen/python/msg/sensor_msgs/point_field_pb2.py +27 -27
- {current_data_py-0.1.4 → current_data_py-0.1.6}/proto_gen/python/msg/sensor_msgs/range_pb2.py +28 -28
- {current_data_py-0.1.4 → current_data_py-0.1.6}/proto_gen/python/msg/sensor_msgs/region_of_interest_pb2.py +25 -25
- {current_data_py-0.1.4 → current_data_py-0.1.6}/proto_gen/python/msg/std_msgs/bool_pb2.py +25 -25
- {current_data_py-0.1.4 → current_data_py-0.1.6}/proto_gen/python/msg/std_msgs/color_rgba_pb2.py +25 -25
- {current_data_py-0.1.4 → current_data_py-0.1.6}/proto_gen/python/msg/std_msgs/float32_pb2.py +25 -25
- {current_data_py-0.1.4 → current_data_py-0.1.6}/proto_gen/python/msg/std_msgs/float64_pb2.py +25 -25
- {current_data_py-0.1.4 → current_data_py-0.1.6}/proto_gen/python/msg/std_msgs/header_pb2.py +26 -26
- {current_data_py-0.1.4 → current_data_py-0.1.6}/proto_gen/python/msg/std_msgs/int32_pb2.py +25 -25
- {current_data_py-0.1.4 → current_data_py-0.1.6}/proto_gen/python/msg/std_msgs/string_pb2.py +25 -25
- {current_data_py-0.1.4 → current_data_py-0.1.6}/proto_gen/python/msg/std_msgs/uint32_pb2.py +25 -25
- {current_data_py-0.1.4 → current_data_py-0.1.6}/proto_gen/python/msg/tf2_msgs/tf_message_pb2.py +26 -26
- {current_data_py-0.1.4 → current_data_py-0.1.6}/pyproject.toml +21 -21
- {current_data_py-0.1.4 → current_data_py-0.1.6}/remote/__init__.py +7 -7
- {current_data_py-0.1.4 → current_data_py-0.1.6}/remote/client.py +83 -62
- {current_data_py-0.1.4 → current_data_py-0.1.6}/remote/source.py +98 -98
- {current_data_py-0.1.4 → current_data_py-0.1.6}/schema/__init__.py +105 -105
- {current_data_py-0.1.4 → current_data_py-0.1.6}/schema/message_metadata.py +30 -30
- {current_data_py-0.1.4 → current_data_py-0.1.6}/setup.py +60 -60
- {current_data_py-0.1.4 → current_data_py-0.1.6}/tests/check_management.py +253 -234
- current_data_py-0.1.6/tests/check_oss.py +317 -0
- {current_data_py-0.1.4 → current_data_py-0.1.6}/tests/check_package.py +43 -43
- {current_data_py-0.1.4 → current_data_py-0.1.6}/tests/check_remote.py +289 -229
- current_data_py-0.1.4/PKG-INFO +0 -43
- current_data_py-0.1.4/README.md +0 -32
- current_data_py-0.1.4/current_data_py.egg-info/PKG-INFO +0 -43
- current_data_py-0.1.4/proto_gen/python/management/service/service_pb2.py +0 -64
- {current_data_py-0.1.4 → current_data_py-0.1.6}/current_data_py.egg-info/dependency_links.txt +0 -0
- {current_data_py-0.1.4 → current_data_py-0.1.6}/current_data_py.egg-info/requires.txt +0 -0
- {current_data_py-0.1.4 → current_data_py-0.1.6}/current_data_py.egg-info/top_level.txt +0 -0
- {current_data_py-0.1.4 → current_data_py-0.1.6}/setup.cfg +0 -0
|
@@ -1,3 +1,4 @@
|
|
|
1
|
-
include tests/check_package.py
|
|
2
|
-
include tests/check_remote.py
|
|
3
|
-
include tests/check_management.py
|
|
1
|
+
include tests/check_package.py
|
|
2
|
+
include tests/check_remote.py
|
|
3
|
+
include tests/check_management.py
|
|
4
|
+
include tests/check_oss.py
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: current-data-py
|
|
3
|
+
Version: 0.1.6
|
|
4
|
+
Summary: Current Data protobuf messages and schema metadata
|
|
5
|
+
Requires-Python: >=3.9
|
|
6
|
+
Description-Content-Type: text/markdown
|
|
7
|
+
Requires-Dist: protobuf>=3.20
|
|
8
|
+
Requires-Dist: turbodata==0.1.0
|
|
9
|
+
Requires-Dist: grpcio>=1.51.1
|
|
10
|
+
Requires-Dist: alibabacloud-oss-v2<2,>=1.4.0
|
|
11
|
+
|
|
12
|
+
# current-data-py
|
|
13
|
+
|
|
14
|
+
## Management
|
|
15
|
+
|
|
16
|
+
`ManagementClient` connects over TLS. Pass `address=` to configure the service,
|
|
17
|
+
or supply your own gRPC channel.
|
|
18
|
+
|
|
19
|
+
The examples below read deployment-specific values from environment variables.
|
|
20
|
+
Set `MANAGEMENT_ADDRESS` to your service address and `UPLOAD_DATA_TYPE` to a valid
|
|
21
|
+
integer enum value supplied by your deployment administrator. Set the bucket and
|
|
22
|
+
object-key variables to resources you are authorized to access. These variables
|
|
23
|
+
are read explicitly by the examples; they are not automatic SDK configuration.
|
|
24
|
+
|
|
25
|
+
```python
|
|
26
|
+
import os
|
|
27
|
+
|
|
28
|
+
from current_data_py.management import ManagementClient, Metadata, MetadataFilter
|
|
29
|
+
|
|
30
|
+
with ManagementClient(address=os.environ["MANAGEMENT_ADDRESS"]) as client:
|
|
31
|
+
upload = client.get_upload_credentials(data_type=int(os.environ["UPLOAD_DATA_TYPE"]))
|
|
32
|
+
read = client.get_read_credentials(bucket=os.environ["READ_BUCKET"])
|
|
33
|
+
|
|
34
|
+
# Register an already uploaded file; replace the example file details.
|
|
35
|
+
record = client.register_data(Metadata(
|
|
36
|
+
name="recording",
|
|
37
|
+
data_type=int(os.environ["UPLOAD_DATA_TYPE"]),
|
|
38
|
+
storages=[{"bucket": os.environ["UPLOADED_BUCKET"], "object_key": os.environ["UPLOADED_OBJECT_KEY"]}],
|
|
39
|
+
file_info={"size_bytes": 1234, "checksum": "sha256:<file-hash>"},
|
|
40
|
+
))
|
|
41
|
+
for item in client.stream_data_references(MetadataFilter(name_prefix="recording")):
|
|
42
|
+
print(item.id, item.storages)
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
`Metadata`, `MetadataFilter`, and responses are Protobuf objects exported by the
|
|
46
|
+
SDK. Registration only saves metadata; it does not upload files. Credential
|
|
47
|
+
calls return fresh credentials without caching. Calls accept `timeout=` and
|
|
48
|
+
`metadata=` for gRPC headers.
|
|
49
|
+
|
|
50
|
+
## Remote TD reads
|
|
51
|
+
|
|
52
|
+
`DataClient.open_td()` accepts an optional `bucket` argument. Without it, reads
|
|
53
|
+
use the management service's `OSS_READ_BUCKET` and `OSS_READ_PREFIX` defaults.
|
|
54
|
+
With it, the SDK requests whole-bucket read credentials, limited by the server's
|
|
55
|
+
RAM role, and selects the first matching storage in the reference. An explicit
|
|
56
|
+
bucket does not inherit the default prefix. Default and per-bucket credentials
|
|
57
|
+
are cached separately and refreshed automatically.
|
|
58
|
+
|
|
59
|
+
```python
|
|
60
|
+
import os
|
|
61
|
+
|
|
62
|
+
from current_data_py.remote import DataClient
|
|
63
|
+
|
|
64
|
+
with DataClient(address=os.environ["MANAGEMENT_ADDRESS"]) as client:
|
|
65
|
+
for reference in client.iter_data():
|
|
66
|
+
# Use the service's default bucket and prefix.
|
|
67
|
+
with client.open_td(reference) as reader:
|
|
68
|
+
print(reader.summary())
|
|
69
|
+
|
|
70
|
+
# Request read access to the selected bucket.
|
|
71
|
+
with client.open_td(reference, bucket=os.environ["READ_BUCKET"]) as reader:
|
|
72
|
+
print(reader.summary())
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
The explicit bucket must contain a storage listed in the reference. This uses
|
|
76
|
+
the existing bucket-aware management API; no protocol changes are required.
|
|
77
|
+
|
|
78
|
+
## OSS
|
|
79
|
+
|
|
80
|
+
```python
|
|
81
|
+
import os
|
|
82
|
+
|
|
83
|
+
from current_data_py.management import ManagementClient
|
|
84
|
+
from current_data_py.oss import OSSClient
|
|
85
|
+
|
|
86
|
+
with ManagementClient(address=os.environ["MANAGEMENT_ADDRESS"]) as management:
|
|
87
|
+
with OSSClient(management, permission="read", bucket=os.environ["READ_BUCKET"]) as source:
|
|
88
|
+
source.download_file(os.environ["SOURCE_OBJECT_KEY"], "/tmp/input.h5")
|
|
89
|
+
|
|
90
|
+
# Convert the input into /tmp/output.td before uploading.
|
|
91
|
+
with OSSClient(management, permission="upload", data_type=int(os.environ["UPLOAD_DATA_TYPE"])) as destination:
|
|
92
|
+
key = destination.object_prefix + "recording/output.td"
|
|
93
|
+
destination.upload_file(key, "/tmp/output.td")
|
|
94
|
+
print(destination.bucket, key) # Use these values when registering metadata.
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
- Read access covers the requested bucket, subject to server permissions.
|
|
98
|
+
The server chooses the upload bucket and prefix based on the requested data type.
|
|
99
|
+
- Credentials refresh automatically. Expired tokens trigger one refresh and retry.
|
|
100
|
+
- Uploads reject overwrites by default; set `forbid_overwrite=False` to replace.
|
|
101
|
+
Downloads replace the local file only on success; its parent directory must exist.
|
|
102
|
+
- Uploads use a single PutObject request ([maximum 5 GiB](https://www.alibabacloud.com/help/en/oss/developer-reference/upload-files-using-oss-sdk-for-python-v2/)); multipart upload is not supported.
|
|
103
|
+
- Supply `region=` for custom upload endpoints. Deploy the current management
|
|
104
|
+
credential API before using the new bucket and data-type arguments.
|
|
105
|
+
|
|
106
|
+
## 打包
|
|
107
|
+
|
|
108
|
+
```bash
|
|
109
|
+
cd current_data_py # 从仓库根目录执行
|
|
110
|
+
# 发布新版本前,更新 pyproject.toml 中的 version
|
|
111
|
+
python -m pip install build twine
|
|
112
|
+
rm -rf build dist
|
|
113
|
+
python -m build
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
## 测试
|
|
117
|
+
|
|
118
|
+
```bash
|
|
119
|
+
python -m twine check dist/*
|
|
120
|
+
python tests/check_distribution.py
|
|
121
|
+
python -m pip install --force-reinstall dist/*.whl
|
|
122
|
+
python -I tests/check_package.py
|
|
123
|
+
python -I tests/check_management.py
|
|
124
|
+
python -I tests/check_remote.py
|
|
125
|
+
python -I tests/check_oss.py
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
## 推送
|
|
129
|
+
|
|
130
|
+
```bash
|
|
131
|
+
# TestPyPI
|
|
132
|
+
python -m twine upload --repository testpypi dist/*
|
|
133
|
+
|
|
134
|
+
# PyPI
|
|
135
|
+
python -m twine upload dist/*
|
|
136
|
+
```
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
# current-data-py
|
|
2
|
+
|
|
3
|
+
## Management
|
|
4
|
+
|
|
5
|
+
`ManagementClient` connects over TLS. Pass `address=` to configure the service,
|
|
6
|
+
or supply your own gRPC channel.
|
|
7
|
+
|
|
8
|
+
The examples below read deployment-specific values from environment variables.
|
|
9
|
+
Set `MANAGEMENT_ADDRESS` to your service address and `UPLOAD_DATA_TYPE` to a valid
|
|
10
|
+
integer enum value supplied by your deployment administrator. Set the bucket and
|
|
11
|
+
object-key variables to resources you are authorized to access. These variables
|
|
12
|
+
are read explicitly by the examples; they are not automatic SDK configuration.
|
|
13
|
+
|
|
14
|
+
```python
|
|
15
|
+
import os
|
|
16
|
+
|
|
17
|
+
from current_data_py.management import ManagementClient, Metadata, MetadataFilter
|
|
18
|
+
|
|
19
|
+
with ManagementClient(address=os.environ["MANAGEMENT_ADDRESS"]) as client:
|
|
20
|
+
upload = client.get_upload_credentials(data_type=int(os.environ["UPLOAD_DATA_TYPE"]))
|
|
21
|
+
read = client.get_read_credentials(bucket=os.environ["READ_BUCKET"])
|
|
22
|
+
|
|
23
|
+
# Register an already uploaded file; replace the example file details.
|
|
24
|
+
record = client.register_data(Metadata(
|
|
25
|
+
name="recording",
|
|
26
|
+
data_type=int(os.environ["UPLOAD_DATA_TYPE"]),
|
|
27
|
+
storages=[{"bucket": os.environ["UPLOADED_BUCKET"], "object_key": os.environ["UPLOADED_OBJECT_KEY"]}],
|
|
28
|
+
file_info={"size_bytes": 1234, "checksum": "sha256:<file-hash>"},
|
|
29
|
+
))
|
|
30
|
+
for item in client.stream_data_references(MetadataFilter(name_prefix="recording")):
|
|
31
|
+
print(item.id, item.storages)
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
`Metadata`, `MetadataFilter`, and responses are Protobuf objects exported by the
|
|
35
|
+
SDK. Registration only saves metadata; it does not upload files. Credential
|
|
36
|
+
calls return fresh credentials without caching. Calls accept `timeout=` and
|
|
37
|
+
`metadata=` for gRPC headers.
|
|
38
|
+
|
|
39
|
+
## Remote TD reads
|
|
40
|
+
|
|
41
|
+
`DataClient.open_td()` accepts an optional `bucket` argument. Without it, reads
|
|
42
|
+
use the management service's `OSS_READ_BUCKET` and `OSS_READ_PREFIX` defaults.
|
|
43
|
+
With it, the SDK requests whole-bucket read credentials, limited by the server's
|
|
44
|
+
RAM role, and selects the first matching storage in the reference. An explicit
|
|
45
|
+
bucket does not inherit the default prefix. Default and per-bucket credentials
|
|
46
|
+
are cached separately and refreshed automatically.
|
|
47
|
+
|
|
48
|
+
```python
|
|
49
|
+
import os
|
|
50
|
+
|
|
51
|
+
from current_data_py.remote import DataClient
|
|
52
|
+
|
|
53
|
+
with DataClient(address=os.environ["MANAGEMENT_ADDRESS"]) as client:
|
|
54
|
+
for reference in client.iter_data():
|
|
55
|
+
# Use the service's default bucket and prefix.
|
|
56
|
+
with client.open_td(reference) as reader:
|
|
57
|
+
print(reader.summary())
|
|
58
|
+
|
|
59
|
+
# Request read access to the selected bucket.
|
|
60
|
+
with client.open_td(reference, bucket=os.environ["READ_BUCKET"]) as reader:
|
|
61
|
+
print(reader.summary())
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
The explicit bucket must contain a storage listed in the reference. This uses
|
|
65
|
+
the existing bucket-aware management API; no protocol changes are required.
|
|
66
|
+
|
|
67
|
+
## OSS
|
|
68
|
+
|
|
69
|
+
```python
|
|
70
|
+
import os
|
|
71
|
+
|
|
72
|
+
from current_data_py.management import ManagementClient
|
|
73
|
+
from current_data_py.oss import OSSClient
|
|
74
|
+
|
|
75
|
+
with ManagementClient(address=os.environ["MANAGEMENT_ADDRESS"]) as management:
|
|
76
|
+
with OSSClient(management, permission="read", bucket=os.environ["READ_BUCKET"]) as source:
|
|
77
|
+
source.download_file(os.environ["SOURCE_OBJECT_KEY"], "/tmp/input.h5")
|
|
78
|
+
|
|
79
|
+
# Convert the input into /tmp/output.td before uploading.
|
|
80
|
+
with OSSClient(management, permission="upload", data_type=int(os.environ["UPLOAD_DATA_TYPE"])) as destination:
|
|
81
|
+
key = destination.object_prefix + "recording/output.td"
|
|
82
|
+
destination.upload_file(key, "/tmp/output.td")
|
|
83
|
+
print(destination.bucket, key) # Use these values when registering metadata.
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
- Read access covers the requested bucket, subject to server permissions.
|
|
87
|
+
The server chooses the upload bucket and prefix based on the requested data type.
|
|
88
|
+
- Credentials refresh automatically. Expired tokens trigger one refresh and retry.
|
|
89
|
+
- Uploads reject overwrites by default; set `forbid_overwrite=False` to replace.
|
|
90
|
+
Downloads replace the local file only on success; its parent directory must exist.
|
|
91
|
+
- Uploads use a single PutObject request ([maximum 5 GiB](https://www.alibabacloud.com/help/en/oss/developer-reference/upload-files-using-oss-sdk-for-python-v2/)); multipart upload is not supported.
|
|
92
|
+
- Supply `region=` for custom upload endpoints. Deploy the current management
|
|
93
|
+
credential API before using the new bucket and data-type arguments.
|
|
94
|
+
|
|
95
|
+
## 打包
|
|
96
|
+
|
|
97
|
+
```bash
|
|
98
|
+
cd current_data_py # 从仓库根目录执行
|
|
99
|
+
# 发布新版本前,更新 pyproject.toml 中的 version
|
|
100
|
+
python -m pip install build twine
|
|
101
|
+
rm -rf build dist
|
|
102
|
+
python -m build
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
## 测试
|
|
106
|
+
|
|
107
|
+
```bash
|
|
108
|
+
python -m twine check dist/*
|
|
109
|
+
python tests/check_distribution.py
|
|
110
|
+
python -m pip install --force-reinstall dist/*.whl
|
|
111
|
+
python -I tests/check_package.py
|
|
112
|
+
python -I tests/check_management.py
|
|
113
|
+
python -I tests/check_remote.py
|
|
114
|
+
python -I tests/check_oss.py
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
## 推送
|
|
118
|
+
|
|
119
|
+
```bash
|
|
120
|
+
# TestPyPI
|
|
121
|
+
python -m twine upload --repository testpypi dist/*
|
|
122
|
+
|
|
123
|
+
# PyPI
|
|
124
|
+
python -m twine upload dist/*
|
|
125
|
+
```
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
"""Protobuf message schemas and metadata for Current Data."""
|
|
2
|
-
|
|
3
|
-
from .schema.message_metadata import descriptor_set, topic_metadata
|
|
4
|
-
from .schema import SCHEMA_TO_CLASS
|
|
5
|
-
|
|
6
|
-
DEFAULT_ADDRESS = "data.current-robotics.work:443"
|
|
7
|
-
|
|
8
|
-
__all__ = ["DEFAULT_ADDRESS", "SCHEMA_TO_CLASS", "descriptor_set", "topic_metadata"]
|
|
1
|
+
"""Protobuf message schemas and metadata for Current Data."""
|
|
2
|
+
|
|
3
|
+
from .schema.message_metadata import descriptor_set, topic_metadata
|
|
4
|
+
from .schema import SCHEMA_TO_CLASS
|
|
5
|
+
|
|
6
|
+
DEFAULT_ADDRESS = "data.current-robotics.work:443"
|
|
7
|
+
|
|
8
|
+
__all__ = ["DEFAULT_ADDRESS", "SCHEMA_TO_CLASS", "descriptor_set", "topic_metadata"]
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: current-data-py
|
|
3
|
+
Version: 0.1.6
|
|
4
|
+
Summary: Current Data protobuf messages and schema metadata
|
|
5
|
+
Requires-Python: >=3.9
|
|
6
|
+
Description-Content-Type: text/markdown
|
|
7
|
+
Requires-Dist: protobuf>=3.20
|
|
8
|
+
Requires-Dist: turbodata==0.1.0
|
|
9
|
+
Requires-Dist: grpcio>=1.51.1
|
|
10
|
+
Requires-Dist: alibabacloud-oss-v2<2,>=1.4.0
|
|
11
|
+
|
|
12
|
+
# current-data-py
|
|
13
|
+
|
|
14
|
+
## Management
|
|
15
|
+
|
|
16
|
+
`ManagementClient` connects over TLS. Pass `address=` to configure the service,
|
|
17
|
+
or supply your own gRPC channel.
|
|
18
|
+
|
|
19
|
+
The examples below read deployment-specific values from environment variables.
|
|
20
|
+
Set `MANAGEMENT_ADDRESS` to your service address and `UPLOAD_DATA_TYPE` to a valid
|
|
21
|
+
integer enum value supplied by your deployment administrator. Set the bucket and
|
|
22
|
+
object-key variables to resources you are authorized to access. These variables
|
|
23
|
+
are read explicitly by the examples; they are not automatic SDK configuration.
|
|
24
|
+
|
|
25
|
+
```python
|
|
26
|
+
import os
|
|
27
|
+
|
|
28
|
+
from current_data_py.management import ManagementClient, Metadata, MetadataFilter
|
|
29
|
+
|
|
30
|
+
with ManagementClient(address=os.environ["MANAGEMENT_ADDRESS"]) as client:
|
|
31
|
+
upload = client.get_upload_credentials(data_type=int(os.environ["UPLOAD_DATA_TYPE"]))
|
|
32
|
+
read = client.get_read_credentials(bucket=os.environ["READ_BUCKET"])
|
|
33
|
+
|
|
34
|
+
# Register an already uploaded file; replace the example file details.
|
|
35
|
+
record = client.register_data(Metadata(
|
|
36
|
+
name="recording",
|
|
37
|
+
data_type=int(os.environ["UPLOAD_DATA_TYPE"]),
|
|
38
|
+
storages=[{"bucket": os.environ["UPLOADED_BUCKET"], "object_key": os.environ["UPLOADED_OBJECT_KEY"]}],
|
|
39
|
+
file_info={"size_bytes": 1234, "checksum": "sha256:<file-hash>"},
|
|
40
|
+
))
|
|
41
|
+
for item in client.stream_data_references(MetadataFilter(name_prefix="recording")):
|
|
42
|
+
print(item.id, item.storages)
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
`Metadata`, `MetadataFilter`, and responses are Protobuf objects exported by the
|
|
46
|
+
SDK. Registration only saves metadata; it does not upload files. Credential
|
|
47
|
+
calls return fresh credentials without caching. Calls accept `timeout=` and
|
|
48
|
+
`metadata=` for gRPC headers.
|
|
49
|
+
|
|
50
|
+
## Remote TD reads
|
|
51
|
+
|
|
52
|
+
`DataClient.open_td()` accepts an optional `bucket` argument. Without it, reads
|
|
53
|
+
use the management service's `OSS_READ_BUCKET` and `OSS_READ_PREFIX` defaults.
|
|
54
|
+
With it, the SDK requests whole-bucket read credentials, limited by the server's
|
|
55
|
+
RAM role, and selects the first matching storage in the reference. An explicit
|
|
56
|
+
bucket does not inherit the default prefix. Default and per-bucket credentials
|
|
57
|
+
are cached separately and refreshed automatically.
|
|
58
|
+
|
|
59
|
+
```python
|
|
60
|
+
import os
|
|
61
|
+
|
|
62
|
+
from current_data_py.remote import DataClient
|
|
63
|
+
|
|
64
|
+
with DataClient(address=os.environ["MANAGEMENT_ADDRESS"]) as client:
|
|
65
|
+
for reference in client.iter_data():
|
|
66
|
+
# Use the service's default bucket and prefix.
|
|
67
|
+
with client.open_td(reference) as reader:
|
|
68
|
+
print(reader.summary())
|
|
69
|
+
|
|
70
|
+
# Request read access to the selected bucket.
|
|
71
|
+
with client.open_td(reference, bucket=os.environ["READ_BUCKET"]) as reader:
|
|
72
|
+
print(reader.summary())
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
The explicit bucket must contain a storage listed in the reference. This uses
|
|
76
|
+
the existing bucket-aware management API; no protocol changes are required.
|
|
77
|
+
|
|
78
|
+
## OSS
|
|
79
|
+
|
|
80
|
+
```python
|
|
81
|
+
import os
|
|
82
|
+
|
|
83
|
+
from current_data_py.management import ManagementClient
|
|
84
|
+
from current_data_py.oss import OSSClient
|
|
85
|
+
|
|
86
|
+
with ManagementClient(address=os.environ["MANAGEMENT_ADDRESS"]) as management:
|
|
87
|
+
with OSSClient(management, permission="read", bucket=os.environ["READ_BUCKET"]) as source:
|
|
88
|
+
source.download_file(os.environ["SOURCE_OBJECT_KEY"], "/tmp/input.h5")
|
|
89
|
+
|
|
90
|
+
# Convert the input into /tmp/output.td before uploading.
|
|
91
|
+
with OSSClient(management, permission="upload", data_type=int(os.environ["UPLOAD_DATA_TYPE"])) as destination:
|
|
92
|
+
key = destination.object_prefix + "recording/output.td"
|
|
93
|
+
destination.upload_file(key, "/tmp/output.td")
|
|
94
|
+
print(destination.bucket, key) # Use these values when registering metadata.
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
- Read access covers the requested bucket, subject to server permissions.
|
|
98
|
+
The server chooses the upload bucket and prefix based on the requested data type.
|
|
99
|
+
- Credentials refresh automatically. Expired tokens trigger one refresh and retry.
|
|
100
|
+
- Uploads reject overwrites by default; set `forbid_overwrite=False` to replace.
|
|
101
|
+
Downloads replace the local file only on success; its parent directory must exist.
|
|
102
|
+
- Uploads use a single PutObject request ([maximum 5 GiB](https://www.alibabacloud.com/help/en/oss/developer-reference/upload-files-using-oss-sdk-for-python-v2/)); multipart upload is not supported.
|
|
103
|
+
- Supply `region=` for custom upload endpoints. Deploy the current management
|
|
104
|
+
credential API before using the new bucket and data-type arguments.
|
|
105
|
+
|
|
106
|
+
## 打包
|
|
107
|
+
|
|
108
|
+
```bash
|
|
109
|
+
cd current_data_py # 从仓库根目录执行
|
|
110
|
+
# 发布新版本前,更新 pyproject.toml 中的 version
|
|
111
|
+
python -m pip install build twine
|
|
112
|
+
rm -rf build dist
|
|
113
|
+
python -m build
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
## 测试
|
|
117
|
+
|
|
118
|
+
```bash
|
|
119
|
+
python -m twine check dist/*
|
|
120
|
+
python tests/check_distribution.py
|
|
121
|
+
python -m pip install --force-reinstall dist/*.whl
|
|
122
|
+
python -I tests/check_package.py
|
|
123
|
+
python -I tests/check_management.py
|
|
124
|
+
python -I tests/check_remote.py
|
|
125
|
+
python -I tests/check_oss.py
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
## 推送
|
|
129
|
+
|
|
130
|
+
```bash
|
|
131
|
+
# TestPyPI
|
|
132
|
+
python -m twine upload --repository testpypi dist/*
|
|
133
|
+
|
|
134
|
+
# PyPI
|
|
135
|
+
python -m twine upload dist/*
|
|
136
|
+
```
|
|
@@ -2,10 +2,12 @@ MANIFEST.in
|
|
|
2
2
|
README.md
|
|
3
3
|
__init__.py
|
|
4
4
|
management.py
|
|
5
|
+
oss.py
|
|
5
6
|
pyproject.toml
|
|
6
7
|
setup.py
|
|
7
8
|
./__init__.py
|
|
8
9
|
./management.py
|
|
10
|
+
./oss.py
|
|
9
11
|
./examples/__init__.py
|
|
10
12
|
./examples/list_schemas.py
|
|
11
13
|
./examples/read_remote_td.py
|
|
@@ -32,5 +34,6 @@ remote/source.py
|
|
|
32
34
|
schema/__init__.py
|
|
33
35
|
schema/message_metadata.py
|
|
34
36
|
tests/check_management.py
|
|
37
|
+
tests/check_oss.py
|
|
35
38
|
tests/check_package.py
|
|
36
39
|
tests/check_remote.py
|
|
@@ -1 +1 @@
|
|
|
1
|
-
"""Runnable examples for Current Data."""
|
|
1
|
+
"""Runnable examples for Current Data."""
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
from ..schema import SCHEMA_TO_CLASS
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
if __name__ == "__main__":
|
|
5
|
-
print("SCHEMA_TO_CLASS:")
|
|
6
|
-
for schema, msg_class in SCHEMA_TO_CLASS.items():
|
|
7
|
-
print(f" {schema}: {msg_class}")
|
|
1
|
+
from ..schema import SCHEMA_TO_CLASS
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
if __name__ == "__main__":
|
|
5
|
+
print("SCHEMA_TO_CLASS:")
|
|
6
|
+
for schema, msg_class in SCHEMA_TO_CLASS.items():
|
|
7
|
+
print(f" {schema}: {msg_class}")
|
|
@@ -1,54 +1,54 @@
|
|
|
1
|
-
"""Read up to ten messages from the first matching remote TD file."""
|
|
2
|
-
|
|
3
|
-
import argparse
|
|
4
|
-
from contextlib import ExitStack, closing
|
|
5
|
-
from itertools import islice
|
|
6
|
-
|
|
7
|
-
import grpc
|
|
8
|
-
from google.protobuf.text_format import MessageToString
|
|
9
|
-
|
|
10
|
-
from ..remote import DataClient, MetadataFilter
|
|
11
|
-
from ..schema import SCHEMA_TO_CLASS
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
def main() -> None:
|
|
15
|
-
parser = argparse.ArgumentParser(description=__doc__)
|
|
16
|
-
parser.add_argument("address", nargs="?", default=None, help="gRPC service address (host:port)")
|
|
17
|
-
parser.add_argument("--insecure", action="store_true", help="Use a plaintext gRPC connection instead of TLS")
|
|
18
|
-
parser.add_argument("--name-prefix", default="", help="Filter recordings by name prefix")
|
|
19
|
-
parser.add_argument("--topic", default="/head/pose/command")
|
|
20
|
-
parser.add_argument("--start-timestamp", type=int, default=0, help="Inclusive start timestamp (ns)")
|
|
21
|
-
parser.add_argument("--end-timestamp", type=int, default=2**63 - 1, help="Inclusive end timestamp (ns)")
|
|
22
|
-
args = parser.parse_args()
|
|
23
|
-
if not 0 <= args.start_timestamp <= args.end_timestamp <= 2**63 - 1:
|
|
24
|
-
parser.error("timestamps must satisfy 0 <= start <= end <= 2**63 - 1")
|
|
25
|
-
|
|
26
|
-
with ExitStack() as stack:
|
|
27
|
-
channel = grpc.insecure_channel(args.address) if args.insecure else None
|
|
28
|
-
if channel is not None:
|
|
29
|
-
stack.callback(channel.close)
|
|
30
|
-
client = stack.enter_context(DataClient(channel, address=args.address))
|
|
31
|
-
with closing(client.iter_data(MetadataFilter(name_prefix=args.name_prefix), timeout=30)) as items:
|
|
32
|
-
metadata = next(items, None)
|
|
33
|
-
if metadata is None:
|
|
34
|
-
print("No matching data")
|
|
35
|
-
return
|
|
36
|
-
|
|
37
|
-
with client.open_td(metadata) as reader:
|
|
38
|
-
schemas = {
|
|
39
|
-
topic.name: topic.metadata["schema_name"]
|
|
40
|
-
for group in reader.summary().topics_infos
|
|
41
|
-
for topic in group.topic_metadatas
|
|
42
|
-
}
|
|
43
|
-
messages = reader.read_messages(
|
|
44
|
-
topic_names=[args.topic],
|
|
45
|
-
start_timestamp=args.start_timestamp,
|
|
46
|
-
end_timestamp=args.end_timestamp,
|
|
47
|
-
)
|
|
48
|
-
for message in islice(messages, 10):
|
|
49
|
-
decoded = SCHEMA_TO_CLASS[schemas[message.topic_name]].FromString(message.data)
|
|
50
|
-
print(metadata.id, message.timestamp, message.topic_name, MessageToString(decoded, as_one_line=True))
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
if __name__ == "__main__":
|
|
54
|
-
main()
|
|
1
|
+
"""Read up to ten messages from the first matching remote TD file."""
|
|
2
|
+
|
|
3
|
+
import argparse
|
|
4
|
+
from contextlib import ExitStack, closing
|
|
5
|
+
from itertools import islice
|
|
6
|
+
|
|
7
|
+
import grpc
|
|
8
|
+
from google.protobuf.text_format import MessageToString
|
|
9
|
+
|
|
10
|
+
from ..remote import DataClient, MetadataFilter
|
|
11
|
+
from ..schema import SCHEMA_TO_CLASS
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def main() -> None:
|
|
15
|
+
parser = argparse.ArgumentParser(description=__doc__)
|
|
16
|
+
parser.add_argument("address", nargs="?", default=None, help="gRPC service address (host:port)")
|
|
17
|
+
parser.add_argument("--insecure", action="store_true", help="Use a plaintext gRPC connection instead of TLS")
|
|
18
|
+
parser.add_argument("--name-prefix", default="", help="Filter recordings by name prefix")
|
|
19
|
+
parser.add_argument("--topic", default="/head/pose/command")
|
|
20
|
+
parser.add_argument("--start-timestamp", type=int, default=0, help="Inclusive start timestamp (ns)")
|
|
21
|
+
parser.add_argument("--end-timestamp", type=int, default=2**63 - 1, help="Inclusive end timestamp (ns)")
|
|
22
|
+
args = parser.parse_args()
|
|
23
|
+
if not 0 <= args.start_timestamp <= args.end_timestamp <= 2**63 - 1:
|
|
24
|
+
parser.error("timestamps must satisfy 0 <= start <= end <= 2**63 - 1")
|
|
25
|
+
|
|
26
|
+
with ExitStack() as stack:
|
|
27
|
+
channel = grpc.insecure_channel(args.address) if args.insecure else None
|
|
28
|
+
if channel is not None:
|
|
29
|
+
stack.callback(channel.close)
|
|
30
|
+
client = stack.enter_context(DataClient(channel, address=args.address))
|
|
31
|
+
with closing(client.iter_data(MetadataFilter(name_prefix=args.name_prefix), timeout=30)) as items:
|
|
32
|
+
metadata = next(items, None)
|
|
33
|
+
if metadata is None:
|
|
34
|
+
print("No matching data")
|
|
35
|
+
return
|
|
36
|
+
|
|
37
|
+
with client.open_td(metadata) as reader:
|
|
38
|
+
schemas = {
|
|
39
|
+
topic.name: topic.metadata["schema_name"]
|
|
40
|
+
for group in reader.summary().topics_infos
|
|
41
|
+
for topic in group.topic_metadatas
|
|
42
|
+
}
|
|
43
|
+
messages = reader.read_messages(
|
|
44
|
+
topic_names=[args.topic],
|
|
45
|
+
start_timestamp=args.start_timestamp,
|
|
46
|
+
end_timestamp=args.end_timestamp,
|
|
47
|
+
)
|
|
48
|
+
for message in islice(messages, 10):
|
|
49
|
+
decoded = SCHEMA_TO_CLASS[schemas[message.topic_name]].FromString(message.data)
|
|
50
|
+
print(metadata.id, message.timestamp, message.topic_name, MessageToString(decoded, as_one_line=True))
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
if __name__ == "__main__":
|
|
54
|
+
main()
|
|
@@ -1,30 +1,30 @@
|
|
|
1
|
-
import argparse
|
|
2
|
-
from pathlib import Path
|
|
3
|
-
|
|
4
|
-
from turbodata import FileReadSource, Reader
|
|
5
|
-
|
|
6
|
-
from ..schema import SCHEMA_TO_CLASS
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
def main() -> None:
|
|
10
|
-
parser = argparse.ArgumentParser(description="Read protobuf messages from a TD file")
|
|
11
|
-
parser.add_argument("path", type=Path)
|
|
12
|
-
args = parser.parse_args()
|
|
13
|
-
|
|
14
|
-
with FileReadSource(str(args.path)) as source:
|
|
15
|
-
reader = Reader(source)
|
|
16
|
-
schemas = {
|
|
17
|
-
topic.name: topic.metadata["schema_name"]
|
|
18
|
-
for group in reader.summary().topics_infos
|
|
19
|
-
for topic in group.topic_metadatas
|
|
20
|
-
}
|
|
21
|
-
|
|
22
|
-
for message in reader.read_messages(topic_names=["/head/pose/command"]):
|
|
23
|
-
schema = schemas[message.topic_name]
|
|
24
|
-
msg = SCHEMA_TO_CLASS[schema]()
|
|
25
|
-
msg.ParseFromString(message.data)
|
|
26
|
-
print(message.timestamp, message.topic_name, schema, msg.position.y)
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
if __name__ == "__main__":
|
|
30
|
-
main()
|
|
1
|
+
import argparse
|
|
2
|
+
from pathlib import Path
|
|
3
|
+
|
|
4
|
+
from turbodata import FileReadSource, Reader
|
|
5
|
+
|
|
6
|
+
from ..schema import SCHEMA_TO_CLASS
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def main() -> None:
|
|
10
|
+
parser = argparse.ArgumentParser(description="Read protobuf messages from a TD file")
|
|
11
|
+
parser.add_argument("path", type=Path)
|
|
12
|
+
args = parser.parse_args()
|
|
13
|
+
|
|
14
|
+
with FileReadSource(str(args.path)) as source:
|
|
15
|
+
reader = Reader(source)
|
|
16
|
+
schemas = {
|
|
17
|
+
topic.name: topic.metadata["schema_name"]
|
|
18
|
+
for group in reader.summary().topics_infos
|
|
19
|
+
for topic in group.topic_metadatas
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
for message in reader.read_messages(topic_names=["/head/pose/command"]):
|
|
23
|
+
schema = schemas[message.topic_name]
|
|
24
|
+
msg = SCHEMA_TO_CLASS[schema]()
|
|
25
|
+
msg.ParseFromString(message.data)
|
|
26
|
+
print(message.timestamp, message.topic_name, schema, msg.position.y)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
if __name__ == "__main__":
|
|
30
|
+
main()
|