current-data-py 0.1.9__tar.gz → 0.1.11__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {current_data_py-0.1.9 → current_data_py-0.1.11}/MANIFEST.in +1 -0
- {current_data_py-0.1.9/current_data_py.egg-info → current_data_py-0.1.11}/PKG-INFO +155 -12
- {current_data_py-0.1.9 → current_data_py-0.1.11}/README.md +154 -11
- {current_data_py-0.1.9 → current_data_py-0.1.11/current_data_py.egg-info}/PKG-INFO +155 -12
- {current_data_py-0.1.9 → current_data_py-0.1.11}/current_data_py.egg-info/SOURCES.txt +3 -0
- current_data_py-0.1.11/labeling.py +102 -0
- current_data_py-0.1.11/management.py +222 -0
- {current_data_py-0.1.9 → current_data_py-0.1.11}/oss.py +25 -7
- current_data_py-0.1.11/proto_gen/python/labeling/service/service_pb2.py +82 -0
- current_data_py-0.1.11/proto_gen/python/labeling/service/service_pb2_grpc.py +433 -0
- current_data_py-0.1.11/proto_gen/python/labeling/task/task_pb2.py +50 -0
- current_data_py-0.1.11/proto_gen/python/labeling/template/template_pb2.py +40 -0
- current_data_py-0.1.11/proto_gen/python/management/selection/selection_pb2.py +27 -0
- current_data_py-0.1.11/proto_gen/python/management/service/service_pb2.py +83 -0
- {current_data_py-0.1.9 → current_data_py-0.1.11}/proto_gen/python/management/service/service_pb2_grpc.py +132 -0
- {current_data_py-0.1.9 → current_data_py-0.1.11}/pyproject.toml +1 -1
- {current_data_py-0.1.9 → current_data_py-0.1.11}/remote/client.py +18 -10
- {current_data_py-0.1.9 → current_data_py-0.1.11}/setup.py +1 -1
- current_data_py-0.1.11/tests/check_labeling.py +111 -0
- current_data_py-0.1.11/tests/check_management.py +454 -0
- {current_data_py-0.1.9 → current_data_py-0.1.11}/tests/check_oss.py +111 -4
- {current_data_py-0.1.9 → current_data_py-0.1.11}/tests/check_remote.py +59 -8
- current_data_py-0.1.9/management.py +0 -118
- current_data_py-0.1.9/proto_gen/python/management/service/service_pb2.py +0 -69
- current_data_py-0.1.9/tests/check_management.py +0 -255
- {current_data_py-0.1.9 → current_data_py-0.1.11}/__init__.py +0 -0
- {current_data_py-0.1.9 → current_data_py-0.1.11}/current_data_py.egg-info/dependency_links.txt +0 -0
- {current_data_py-0.1.9 → current_data_py-0.1.11}/current_data_py.egg-info/requires.txt +0 -0
- {current_data_py-0.1.9 → current_data_py-0.1.11}/current_data_py.egg-info/top_level.txt +0 -0
- {current_data_py-0.1.9 → current_data_py-0.1.11}/decoding.py +0 -0
- {current_data_py-0.1.9 → current_data_py-0.1.11}/examples/__init__.py +0 -0
- {current_data_py-0.1.9 → current_data_py-0.1.11}/examples/list_schemas.py +0 -0
- {current_data_py-0.1.9 → current_data_py-0.1.11}/examples/read_remote_td.py +0 -0
- {current_data_py-0.1.9 → current_data_py-0.1.11}/examples/read_td.py +0 -0
- {current_data_py-0.1.9 → current_data_py-0.1.11}/examples/write_td.py +0 -0
- {current_data_py-0.1.9 → current_data_py-0.1.11}/proto_gen/python/management/metadata/metadata_pb2.py +0 -0
- {current_data_py-0.1.9 → current_data_py-0.1.11}/proto_gen/python/msg/builtin_interfaces/duration_pb2.py +0 -0
- {current_data_py-0.1.9 → current_data_py-0.1.11}/proto_gen/python/msg/builtin_interfaces/time_pb2.py +0 -0
- {current_data_py-0.1.9 → current_data_py-0.1.11}/proto_gen/python/msg/custom_msgs/joint_current_pb2.py +0 -0
- {current_data_py-0.1.9 → current_data_py-0.1.11}/proto_gen/python/msg/custom_msgs/pose_rpy_pb2.py +0 -0
- {current_data_py-0.1.9 → current_data_py-0.1.11}/proto_gen/python/msg/custom_msgs/slam_quality_pb2.py +0 -0
- {current_data_py-0.1.9 → current_data_py-0.1.11}/proto_gen/python/msg/custom_msgs/sonic_frame_pb2.py +0 -0
- {current_data_py-0.1.9 → current_data_py-0.1.11}/proto_gen/python/msg/custom_msgs/target_quality_pb2.py +0 -0
- {current_data_py-0.1.9 → current_data_py-0.1.11}/proto_gen/python/msg/foxglove/compressed_audio_pb2.py +0 -0
- {current_data_py-0.1.9 → current_data_py-0.1.11}/proto_gen/python/msg/foxglove/compressed_video_pb2.py +0 -0
- {current_data_py-0.1.9 → current_data_py-0.1.11}/proto_gen/python/msg/geometry_msgs/point32_pb2.py +0 -0
- {current_data_py-0.1.9 → current_data_py-0.1.11}/proto_gen/python/msg/geometry_msgs/point_pb2.py +0 -0
- {current_data_py-0.1.9 → current_data_py-0.1.11}/proto_gen/python/msg/geometry_msgs/pose_pb2.py +0 -0
- {current_data_py-0.1.9 → current_data_py-0.1.11}/proto_gen/python/msg/geometry_msgs/pose_stamped_pb2.py +0 -0
- {current_data_py-0.1.9 → current_data_py-0.1.11}/proto_gen/python/msg/geometry_msgs/pose_with_covariance_pb2.py +0 -0
- {current_data_py-0.1.9 → current_data_py-0.1.11}/proto_gen/python/msg/geometry_msgs/pose_with_covariance_stamped_pb2.py +0 -0
- {current_data_py-0.1.9 → current_data_py-0.1.11}/proto_gen/python/msg/geometry_msgs/quaternion_pb2.py +0 -0
- {current_data_py-0.1.9 → current_data_py-0.1.11}/proto_gen/python/msg/geometry_msgs/transform_pb2.py +0 -0
- {current_data_py-0.1.9 → current_data_py-0.1.11}/proto_gen/python/msg/geometry_msgs/transform_stamped_pb2.py +0 -0
- {current_data_py-0.1.9 → current_data_py-0.1.11}/proto_gen/python/msg/geometry_msgs/twist_pb2.py +0 -0
- {current_data_py-0.1.9 → current_data_py-0.1.11}/proto_gen/python/msg/geometry_msgs/twist_stamped_pb2.py +0 -0
- {current_data_py-0.1.9 → current_data_py-0.1.11}/proto_gen/python/msg/geometry_msgs/twist_with_covariance_pb2.py +0 -0
- {current_data_py-0.1.9 → current_data_py-0.1.11}/proto_gen/python/msg/geometry_msgs/twist_with_covariance_stamped_pb2.py +0 -0
- {current_data_py-0.1.9 → current_data_py-0.1.11}/proto_gen/python/msg/geometry_msgs/vector3_pb2.py +0 -0
- {current_data_py-0.1.9 → current_data_py-0.1.11}/proto_gen/python/msg/geometry_msgs/wrench_pb2.py +0 -0
- {current_data_py-0.1.9 → current_data_py-0.1.11}/proto_gen/python/msg/geometry_msgs/wrench_stamped_pb2.py +0 -0
- {current_data_py-0.1.9 → current_data_py-0.1.11}/proto_gen/python/msg/nav_msgs/map_meta_data_pb2.py +0 -0
- {current_data_py-0.1.9 → current_data_py-0.1.11}/proto_gen/python/msg/nav_msgs/occupancy_grid_pb2.py +0 -0
- {current_data_py-0.1.9 → current_data_py-0.1.11}/proto_gen/python/msg/nav_msgs/odometry_pb2.py +0 -0
- {current_data_py-0.1.9 → current_data_py-0.1.11}/proto_gen/python/msg/nav_msgs/path_pb2.py +0 -0
- {current_data_py-0.1.9 → current_data_py-0.1.11}/proto_gen/python/msg/sensor_msgs/battery_state_pb2.py +0 -0
- {current_data_py-0.1.9 → current_data_py-0.1.11}/proto_gen/python/msg/sensor_msgs/camera_info_pb2.py +0 -0
- {current_data_py-0.1.9 → current_data_py-0.1.11}/proto_gen/python/msg/sensor_msgs/compressed_image_pb2.py +0 -0
- {current_data_py-0.1.9 → current_data_py-0.1.11}/proto_gen/python/msg/sensor_msgs/image_pb2.py +0 -0
- {current_data_py-0.1.9 → current_data_py-0.1.11}/proto_gen/python/msg/sensor_msgs/imu_pb2.py +0 -0
- {current_data_py-0.1.9 → current_data_py-0.1.11}/proto_gen/python/msg/sensor_msgs/joint_state_pb2.py +0 -0
- {current_data_py-0.1.9 → current_data_py-0.1.11}/proto_gen/python/msg/sensor_msgs/laser_scan_pb2.py +0 -0
- {current_data_py-0.1.9 → current_data_py-0.1.11}/proto_gen/python/msg/sensor_msgs/nav_sat_fix_pb2.py +0 -0
- {current_data_py-0.1.9 → current_data_py-0.1.11}/proto_gen/python/msg/sensor_msgs/nav_sat_status_pb2.py +0 -0
- {current_data_py-0.1.9 → current_data_py-0.1.11}/proto_gen/python/msg/sensor_msgs/point_cloud2_pb2.py +0 -0
- {current_data_py-0.1.9 → current_data_py-0.1.11}/proto_gen/python/msg/sensor_msgs/point_field_pb2.py +0 -0
- {current_data_py-0.1.9 → current_data_py-0.1.11}/proto_gen/python/msg/sensor_msgs/range_pb2.py +0 -0
- {current_data_py-0.1.9 → current_data_py-0.1.11}/proto_gen/python/msg/sensor_msgs/region_of_interest_pb2.py +0 -0
- {current_data_py-0.1.9 → current_data_py-0.1.11}/proto_gen/python/msg/std_msgs/bool_pb2.py +0 -0
- {current_data_py-0.1.9 → current_data_py-0.1.11}/proto_gen/python/msg/std_msgs/color_rgba_pb2.py +0 -0
- {current_data_py-0.1.9 → current_data_py-0.1.11}/proto_gen/python/msg/std_msgs/float32_pb2.py +0 -0
- {current_data_py-0.1.9 → current_data_py-0.1.11}/proto_gen/python/msg/std_msgs/float64_pb2.py +0 -0
- {current_data_py-0.1.9 → current_data_py-0.1.11}/proto_gen/python/msg/std_msgs/header_pb2.py +0 -0
- {current_data_py-0.1.9 → current_data_py-0.1.11}/proto_gen/python/msg/std_msgs/int32_pb2.py +0 -0
- {current_data_py-0.1.9 → current_data_py-0.1.11}/proto_gen/python/msg/std_msgs/string_pb2.py +0 -0
- {current_data_py-0.1.9 → current_data_py-0.1.11}/proto_gen/python/msg/std_msgs/uint32_pb2.py +0 -0
- {current_data_py-0.1.9 → current_data_py-0.1.11}/proto_gen/python/msg/tf2_msgs/tf_message_pb2.py +0 -0
- {current_data_py-0.1.9 → current_data_py-0.1.11}/remote/__init__.py +0 -0
- {current_data_py-0.1.9 → current_data_py-0.1.11}/remote/source.py +0 -0
- {current_data_py-0.1.9 → current_data_py-0.1.11}/schema/__init__.py +0 -0
- {current_data_py-0.1.9 → current_data_py-0.1.11}/schema/message_metadata.py +0 -0
- {current_data_py-0.1.9 → current_data_py-0.1.11}/setup.cfg +0 -0
- {current_data_py-0.1.9 → current_data_py-0.1.11}/tests/check_environment.py +0 -0
- {current_data_py-0.1.9 → current_data_py-0.1.11}/tests/check_package.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: current-data-py
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.11
|
|
4
4
|
Summary: Current Data protobuf messages and schema metadata
|
|
5
5
|
Requires-Python: >=3.9
|
|
6
6
|
Description-Content-Type: text/markdown
|
|
@@ -83,6 +83,97 @@ SDK. Registration only saves metadata; it does not upload files. Credential
|
|
|
83
83
|
calls return fresh credentials without caching. Calls accept `timeout=` and
|
|
84
84
|
`metadata=` for gRPC headers.
|
|
85
85
|
|
|
86
|
+
For uploads that distinguish raw and converted data, use V2 with both arguments:
|
|
87
|
+
|
|
88
|
+
```python
|
|
89
|
+
with ManagementClient(address=os.environ["MANAGEMENT_ADDRESS"]) as client:
|
|
90
|
+
raw = client.get_upload_credentials_v2(data_type="YAM", data_kind="RAW")
|
|
91
|
+
converted = client.get_upload_credentials_v2(
|
|
92
|
+
data_type="YAM", data_kind="CONVERTED", use_internal_endpoint=True,
|
|
93
|
+
)
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
V2 accepts `EGOSUITE`, `YAM`, and `CURRENT_V2` data types and `RAW` or `CONVERTED`
|
|
97
|
+
data kinds. Each argument also accepts its full enum name (`DATA_TYPE_YAM`,
|
|
98
|
+
`DATA_KIND_RAW`) or integer value. `DataKind` is exported from
|
|
99
|
+
`current_data_py.management`; data type constants are in `metadata_pb2`.
|
|
100
|
+
Unspecified or unknown enum values are rejected. `RAW` is not a V2 data type.
|
|
101
|
+
The server must have the selected environment/type/kind destination configured;
|
|
102
|
+
unfilled placeholders return `FAILED_PRECONDITION`. V2 requires a deployed
|
|
103
|
+
`GetUploadCredentialsV2` RPC and never falls back to the old RPC on failure.
|
|
104
|
+
The existing `get_upload_credentials()` API and its defaults remain unchanged.
|
|
105
|
+
V2 responses include the selected destination's `oss_endpoint` and `region`,
|
|
106
|
+
along with its bucket and prefix; raw and converted destinations may use different
|
|
107
|
+
OSS regions. The server derives the standard Alibaba Cloud endpoint from `region`:
|
|
108
|
+
`https://oss-<region>.aliyuncs.com` by default, or
|
|
109
|
+
`https://oss-<region>-internal.aliyuncs.com` with `use_internal_endpoint=True`.
|
|
110
|
+
Use the internal endpoint from Alibaba Cloud resources in the same region.
|
|
111
|
+
See the [Alibaba Cloud endpoint reference](https://www.alibabacloud.com/help/en/oss/user-guide/regions-and-endpoints).
|
|
112
|
+
`OSSClient` accepts the same option for V2 uploads and retains it when refreshing
|
|
113
|
+
credentials. It uses the returned endpoint and region for connections and signing.
|
|
114
|
+
An explicit `region=` must match the returned region. If the endpoint or
|
|
115
|
+
region changes during credential refresh, create a new `OSSClient`.
|
|
116
|
+
|
|
117
|
+
For reads in the bucket's own region, use the new read V2 RPC:
|
|
118
|
+
|
|
119
|
+
```python
|
|
120
|
+
with ManagementClient(address=os.environ["MANAGEMENT_ADDRESS"]) as client:
|
|
121
|
+
read = client.get_read_credentials_v2(
|
|
122
|
+
bucket=os.environ["READ_BUCKET"], use_internal_endpoint=False,
|
|
123
|
+
)
|
|
124
|
+
```
|
|
125
|
+
|
|
126
|
+
The bucket is required and must appear in the service's environment-specific
|
|
127
|
+
bucket-to-region map, shared with V2 uploads. Reads grant whole-bucket `GetObject`
|
|
128
|
+
access bounded by the read RAM role. Public networking is the default; set
|
|
129
|
+
`use_internal_endpoint=True` for internal networking. Unconfigured buckets return
|
|
130
|
+
`FAILED_PRECONDITION`; empty or malformed buckets return `INVALID_ARGUMENT`.
|
|
131
|
+
The existing `get_read_credentials()` API keeps its configured default endpoint,
|
|
132
|
+
region and scope. V2 calls never fall back to it on failure.
|
|
133
|
+
|
|
134
|
+
### Multi-cloud credentials (V3)
|
|
135
|
+
|
|
136
|
+
Use V3 to explicitly choose OSS or native TOS. `StorageProvider` and
|
|
137
|
+
`StorageCredentials` are exported from `current_data_py.management`.
|
|
138
|
+
|
|
139
|
+
```python
|
|
140
|
+
from current_data_py.management import ManagementClient
|
|
141
|
+
|
|
142
|
+
with ManagementClient() as client:
|
|
143
|
+
upload = client.get_upload_credentials_v3(
|
|
144
|
+
provider="TOS", data_type="YAM", data_kind="RAW",
|
|
145
|
+
use_internal_endpoint=False,
|
|
146
|
+
)
|
|
147
|
+
read = client.get_read_credentials_v3(
|
|
148
|
+
provider="TOS", bucket=upload.bucket,
|
|
149
|
+
use_internal_endpoint=False,
|
|
150
|
+
)
|
|
151
|
+
print(upload.provider, upload.bucket, upload.endpoint, upload.region)
|
|
152
|
+
```
|
|
153
|
+
|
|
154
|
+
`provider` is required and accepts `OSS`/`TOS`, full enum names such as
|
|
155
|
+
`STORAGE_PROVIDER_TOS`, or enum integer values. Unspecified/unknown providers
|
|
156
|
+
are rejected. Uploads require the same explicit type/kind as V2; reads require
|
|
157
|
+
an explicitly configured bucket on the selected provider. OSS V3 uses the same
|
|
158
|
+
routes and roles as OSS V2. TOS configuration and IAM preparation are described
|
|
159
|
+
in the [service deployment guide](../management_service/README.md#tos-preparation-for-v3).
|
|
160
|
+
|
|
161
|
+
Both calls return `provider`, `endpoint`, `region`, `bucket`, `object_prefix`,
|
|
162
|
+
`access_key_id`, `access_key_secret`, `security_token`, and `expires_at`.
|
|
163
|
+
`object_prefix` is empty for whole-bucket reads. Use `endpoint` instead of V2's
|
|
164
|
+
`oss_endpoint`. TOS public endpoints use `tos-<region>.volces.com`; internal
|
|
165
|
+
endpoints use `tos-<region>.ivolces.com`. Internal networking requires access to
|
|
166
|
+
the destination cloud and region's private network.
|
|
167
|
+
|
|
168
|
+
V3 calls accept `timeout` and `metadata`, do not cache credentials, and never
|
|
169
|
+
retry or fall back to earlier RPC versions. Old servers return `UNIMPLEMENTED`;
|
|
170
|
+
unconfigured destinations return `FAILED_PRECONDITION`. A response whose
|
|
171
|
+
provider differs from the request raises `ValueError`.
|
|
172
|
+
|
|
173
|
+
These methods only fetch credentials. Pass TOS credentials to a native TOS
|
|
174
|
+
client and refresh them before expiration. `OSSClient`, `DataClient`, metadata
|
|
175
|
+
registration and signed URLs have not been extended to TOS by this change.
|
|
176
|
+
|
|
86
177
|
## Remote TD reads
|
|
87
178
|
|
|
88
179
|
### Payload decoding
|
|
@@ -183,10 +274,12 @@ to each call.
|
|
|
183
274
|
|
|
184
275
|
`DataClient.open_td()` accepts an optional `bucket` argument. Without it, reads
|
|
185
276
|
use the management service's `OSS_READ_BUCKET` and `OSS_READ_PREFIX` defaults.
|
|
186
|
-
With it, the SDK
|
|
187
|
-
RAM role, and selects the first matching
|
|
188
|
-
bucket does not inherit the default prefix.
|
|
189
|
-
|
|
277
|
+
With it, the SDK uses `GetReadCredentialsV2` for whole-bucket credentials in the
|
|
278
|
+
bucket's region, limited by the server's RAM role, and selects the first matching
|
|
279
|
+
storage in the reference. An explicit bucket does not inherit the default prefix.
|
|
280
|
+
Set `use_internal_endpoint=True` with an explicit bucket to use internal networking.
|
|
281
|
+
Credentials are cached separately for each `(bucket, use_internal_endpoint)` pair,
|
|
282
|
+
apart from the legacy default scope, and refreshed automatically.
|
|
190
283
|
|
|
191
284
|
```python
|
|
192
285
|
import os
|
|
@@ -204,8 +297,10 @@ with DataClient(address=os.environ["MANAGEMENT_ADDRESS"]) as client:
|
|
|
204
297
|
print(reader.summary())
|
|
205
298
|
```
|
|
206
299
|
|
|
207
|
-
The explicit bucket must contain a storage listed in the reference
|
|
208
|
-
|
|
300
|
+
The explicit bucket must contain a storage listed in the reference and have a
|
|
301
|
+
configured region on the server. Deploy `GetReadCredentialsV2` before upgrading
|
|
302
|
+
clients that use explicit buckets; older servers return `UNIMPLEMENTED` and the
|
|
303
|
+
SDK does not fall back to a potentially incorrect legacy endpoint.
|
|
209
304
|
|
|
210
305
|
## OSS
|
|
211
306
|
|
|
@@ -220,20 +315,67 @@ with ManagementClient(address=os.environ["MANAGEMENT_ADDRESS"]) as management:
|
|
|
220
315
|
source.download_file(os.environ["SOURCE_OBJECT_KEY"], "/tmp/input.h5")
|
|
221
316
|
|
|
222
317
|
# Convert the input into /tmp/output.td before uploading.
|
|
223
|
-
with OSSClient(management, permission="upload", data_type=int(os.environ["UPLOAD_DATA_TYPE"])
|
|
318
|
+
with OSSClient(management, permission="upload", data_type=int(os.environ["UPLOAD_DATA_TYPE"]),
|
|
319
|
+
data_kind="CONVERTED") as destination:
|
|
224
320
|
key = destination.object_prefix + "recording/output.td"
|
|
225
321
|
destination.upload_file(key, "/tmp/output.td")
|
|
226
322
|
print(destination.bucket, key) # Use these values when registering metadata.
|
|
227
323
|
```
|
|
228
324
|
|
|
229
|
-
- Read access covers the requested bucket, subject to server permissions.
|
|
230
|
-
The server chooses the upload bucket and prefix based on the requested data type
|
|
325
|
+
- Read access uses V2 and covers the requested bucket, subject to server permissions.
|
|
326
|
+
The server chooses the upload bucket and prefix based on the requested data type
|
|
327
|
+
and kind. Supplying `data_kind` selects V2 and requires `data_type`; omit
|
|
328
|
+
`data_kind` to retain the legacy upload API. Read clients reject both arguments.
|
|
329
|
+
- Reads and V2 uploads use the public endpoint by default. Set `use_internal_endpoint=True`
|
|
330
|
+
on `OSSClient` to select internal networking. Legacy uploads reject this option.
|
|
231
331
|
- Credentials refresh automatically. Expired tokens trigger one refresh and retry.
|
|
232
332
|
- Uploads reject overwrites by default; set `forbid_overwrite=False` to replace.
|
|
233
333
|
Downloads replace the local file only on success; its parent directory must exist.
|
|
234
334
|
- Uploads use a single PutObject request ([maximum 5 GiB](https://www.alibabacloud.com/help/en/oss/developer-reference/upload-files-using-oss-sdk-for-python-v2/)); multipart upload is not supported.
|
|
235
|
-
- Supply `region=` for custom upload endpoints
|
|
236
|
-
|
|
335
|
+
- Supply `region=` for legacy custom upload endpoints; V2 returns its own region.
|
|
336
|
+
Deploy the V2 credential APIs and configure bucket regions before using reads
|
|
337
|
+
or the upload `data_kind` option.
|
|
338
|
+
|
|
339
|
+
## Pre-annotation
|
|
340
|
+
|
|
341
|
+
`current_data_py.labeling.LabelingClient` wraps the pre-annotation RPCs
|
|
342
|
+
`CreateTask`, `ListTasks`, `ListItems`, `WriteResult`, and `UpdatePreAnnotation` with
|
|
343
|
+
ManagementClient's TLS, package default address, caller-owned channel, per-call
|
|
344
|
+
`timeout` (30 seconds by default, `None` disables it), and `metadata` conventions.
|
|
345
|
+
Errors propagate without retries. Protobuf types are available through
|
|
346
|
+
`service_pb2`, `task_pb2`, and `DataSelection` in the same module.
|
|
347
|
+
|
|
348
|
+
```python
|
|
349
|
+
from current_data_py.labeling import LabelingClient, service_pb2, task_pb2
|
|
350
|
+
|
|
351
|
+
with LabelingClient() as labeling:
|
|
352
|
+
task = labeling.create_task(
|
|
353
|
+
"template-id", name="Pre-annotation",
|
|
354
|
+
input_sources=[task_pb2.InputSource(td_file=task_pb2.TdFileSource(data_id="data-id"))],
|
|
355
|
+
start_pre_annotation=True,
|
|
356
|
+
)
|
|
357
|
+
for item in labeling.list_items(task.task_id):
|
|
358
|
+
# Match question keys and option keys to the existing built-in template.
|
|
359
|
+
labeling.write_result(item.item_id, task_pb2.Result(question_results=[
|
|
360
|
+
task_pb2.QuestionResult(
|
|
361
|
+
question_key="quality",
|
|
362
|
+
choice=task_pb2.ChoiceResult(selected_option_keys=["ok"]),
|
|
363
|
+
),
|
|
364
|
+
]))
|
|
365
|
+
# On inference failure instead: write_result(item.item_id, pre_annotation_error="reason")
|
|
366
|
+
labeling.update_pre_annotation(
|
|
367
|
+
task.task_id, service_pb2.UpdatePreAnnotationRequest.ACTION_OPEN_HUMAN_REVIEW,
|
|
368
|
+
)
|
|
369
|
+
```
|
|
370
|
+
|
|
371
|
+
Alternatively, pass `data_selection=DataSelection(...)` when creating a task.
|
|
372
|
+
Use `ACTION_ACCEPT_ALL_AND_COMPLETE` to accept all pre-annotations as final labels.
|
|
373
|
+
Use an existing built-in template; custom templates do not support pre-annotation.
|
|
374
|
+
The server validates inputs and task transitions.
|
|
375
|
+
|
|
376
|
+
`list_tasks` and `list_items` return repeated protobuf collections.
|
|
377
|
+
`create_task` and `update_pre_annotation` return full protobuf responses
|
|
378
|
+
(including item count and task ID, or status). `write_result` returns `None`.
|
|
237
379
|
|
|
238
380
|
## 打包
|
|
239
381
|
|
|
@@ -254,6 +396,7 @@ python -m pip install --force-reinstall dist/*.whl
|
|
|
254
396
|
python -I tests/check_package.py
|
|
255
397
|
python -I tests/check_environment.py
|
|
256
398
|
python -I tests/check_management.py
|
|
399
|
+
python -I tests/check_labeling.py
|
|
257
400
|
python -I tests/check_remote.py
|
|
258
401
|
python -I tests/check_oss.py
|
|
259
402
|
```
|
|
@@ -69,6 +69,97 @@ SDK. Registration only saves metadata; it does not upload files. Credential
|
|
|
69
69
|
calls return fresh credentials without caching. Calls accept `timeout=` and
|
|
70
70
|
`metadata=` for gRPC headers.
|
|
71
71
|
|
|
72
|
+
For uploads that distinguish raw and converted data, use V2 with both arguments:
|
|
73
|
+
|
|
74
|
+
```python
|
|
75
|
+
with ManagementClient(address=os.environ["MANAGEMENT_ADDRESS"]) as client:
|
|
76
|
+
raw = client.get_upload_credentials_v2(data_type="YAM", data_kind="RAW")
|
|
77
|
+
converted = client.get_upload_credentials_v2(
|
|
78
|
+
data_type="YAM", data_kind="CONVERTED", use_internal_endpoint=True,
|
|
79
|
+
)
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
V2 accepts `EGOSUITE`, `YAM`, and `CURRENT_V2` data types and `RAW` or `CONVERTED`
|
|
83
|
+
data kinds. Each argument also accepts its full enum name (`DATA_TYPE_YAM`,
|
|
84
|
+
`DATA_KIND_RAW`) or integer value. `DataKind` is exported from
|
|
85
|
+
`current_data_py.management`; data type constants are in `metadata_pb2`.
|
|
86
|
+
Unspecified or unknown enum values are rejected. `RAW` is not a V2 data type.
|
|
87
|
+
The server must have the selected environment/type/kind destination configured;
|
|
88
|
+
unfilled placeholders return `FAILED_PRECONDITION`. V2 requires a deployed
|
|
89
|
+
`GetUploadCredentialsV2` RPC and never falls back to the old RPC on failure.
|
|
90
|
+
The existing `get_upload_credentials()` API and its defaults remain unchanged.
|
|
91
|
+
V2 responses include the selected destination's `oss_endpoint` and `region`,
|
|
92
|
+
along with its bucket and prefix; raw and converted destinations may use different
|
|
93
|
+
OSS regions. The server derives the standard Alibaba Cloud endpoint from `region`:
|
|
94
|
+
`https://oss-<region>.aliyuncs.com` by default, or
|
|
95
|
+
`https://oss-<region>-internal.aliyuncs.com` with `use_internal_endpoint=True`.
|
|
96
|
+
Use the internal endpoint from Alibaba Cloud resources in the same region.
|
|
97
|
+
See the [Alibaba Cloud endpoint reference](https://www.alibabacloud.com/help/en/oss/user-guide/regions-and-endpoints).
|
|
98
|
+
`OSSClient` accepts the same option for V2 uploads and retains it when refreshing
|
|
99
|
+
credentials. It uses the returned endpoint and region for connections and signing.
|
|
100
|
+
An explicit `region=` must match the returned region. If the endpoint or
|
|
101
|
+
region changes during credential refresh, create a new `OSSClient`.
|
|
102
|
+
|
|
103
|
+
For reads in the bucket's own region, use the new read V2 RPC:
|
|
104
|
+
|
|
105
|
+
```python
|
|
106
|
+
with ManagementClient(address=os.environ["MANAGEMENT_ADDRESS"]) as client:
|
|
107
|
+
read = client.get_read_credentials_v2(
|
|
108
|
+
bucket=os.environ["READ_BUCKET"], use_internal_endpoint=False,
|
|
109
|
+
)
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
The bucket is required and must appear in the service's environment-specific
|
|
113
|
+
bucket-to-region map, shared with V2 uploads. Reads grant whole-bucket `GetObject`
|
|
114
|
+
access bounded by the read RAM role. Public networking is the default; set
|
|
115
|
+
`use_internal_endpoint=True` for internal networking. Unconfigured buckets return
|
|
116
|
+
`FAILED_PRECONDITION`; empty or malformed buckets return `INVALID_ARGUMENT`.
|
|
117
|
+
The existing `get_read_credentials()` API keeps its configured default endpoint,
|
|
118
|
+
region and scope. V2 calls never fall back to it on failure.
|
|
119
|
+
|
|
120
|
+
### Multi-cloud credentials (V3)
|
|
121
|
+
|
|
122
|
+
Use V3 to explicitly choose OSS or native TOS. `StorageProvider` and
|
|
123
|
+
`StorageCredentials` are exported from `current_data_py.management`.
|
|
124
|
+
|
|
125
|
+
```python
|
|
126
|
+
from current_data_py.management import ManagementClient
|
|
127
|
+
|
|
128
|
+
with ManagementClient() as client:
|
|
129
|
+
upload = client.get_upload_credentials_v3(
|
|
130
|
+
provider="TOS", data_type="YAM", data_kind="RAW",
|
|
131
|
+
use_internal_endpoint=False,
|
|
132
|
+
)
|
|
133
|
+
read = client.get_read_credentials_v3(
|
|
134
|
+
provider="TOS", bucket=upload.bucket,
|
|
135
|
+
use_internal_endpoint=False,
|
|
136
|
+
)
|
|
137
|
+
print(upload.provider, upload.bucket, upload.endpoint, upload.region)
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
`provider` is required and accepts `OSS`/`TOS`, full enum names such as
|
|
141
|
+
`STORAGE_PROVIDER_TOS`, or enum integer values. Unspecified/unknown providers
|
|
142
|
+
are rejected. Uploads require the same explicit type/kind as V2; reads require
|
|
143
|
+
an explicitly configured bucket on the selected provider. OSS V3 uses the same
|
|
144
|
+
routes and roles as OSS V2. TOS configuration and IAM preparation are described
|
|
145
|
+
in the [service deployment guide](../management_service/README.md#tos-preparation-for-v3).
|
|
146
|
+
|
|
147
|
+
Both calls return `provider`, `endpoint`, `region`, `bucket`, `object_prefix`,
|
|
148
|
+
`access_key_id`, `access_key_secret`, `security_token`, and `expires_at`.
|
|
149
|
+
`object_prefix` is empty for whole-bucket reads. Use `endpoint` instead of V2's
|
|
150
|
+
`oss_endpoint`. TOS public endpoints use `tos-<region>.volces.com`; internal
|
|
151
|
+
endpoints use `tos-<region>.ivolces.com`. Internal networking requires access to
|
|
152
|
+
the destination cloud and region's private network.
|
|
153
|
+
|
|
154
|
+
V3 calls accept `timeout` and `metadata`, do not cache credentials, and never
|
|
155
|
+
retry or fall back to earlier RPC versions. Old servers return `UNIMPLEMENTED`;
|
|
156
|
+
unconfigured destinations return `FAILED_PRECONDITION`. A response whose
|
|
157
|
+
provider differs from the request raises `ValueError`.
|
|
158
|
+
|
|
159
|
+
These methods only fetch credentials. Pass TOS credentials to a native TOS
|
|
160
|
+
client and refresh them before expiration. `OSSClient`, `DataClient`, metadata
|
|
161
|
+
registration and signed URLs have not been extended to TOS by this change.
|
|
162
|
+
|
|
72
163
|
## Remote TD reads
|
|
73
164
|
|
|
74
165
|
### Payload decoding
|
|
@@ -169,10 +260,12 @@ to each call.
|
|
|
169
260
|
|
|
170
261
|
`DataClient.open_td()` accepts an optional `bucket` argument. Without it, reads
|
|
171
262
|
use the management service's `OSS_READ_BUCKET` and `OSS_READ_PREFIX` defaults.
|
|
172
|
-
With it, the SDK
|
|
173
|
-
RAM role, and selects the first matching
|
|
174
|
-
bucket does not inherit the default prefix.
|
|
175
|
-
|
|
263
|
+
With it, the SDK uses `GetReadCredentialsV2` for whole-bucket credentials in the
|
|
264
|
+
bucket's region, limited by the server's RAM role, and selects the first matching
|
|
265
|
+
storage in the reference. An explicit bucket does not inherit the default prefix.
|
|
266
|
+
Set `use_internal_endpoint=True` with an explicit bucket to use internal networking.
|
|
267
|
+
Credentials are cached separately for each `(bucket, use_internal_endpoint)` pair,
|
|
268
|
+
apart from the legacy default scope, and refreshed automatically.
|
|
176
269
|
|
|
177
270
|
```python
|
|
178
271
|
import os
|
|
@@ -190,8 +283,10 @@ with DataClient(address=os.environ["MANAGEMENT_ADDRESS"]) as client:
|
|
|
190
283
|
print(reader.summary())
|
|
191
284
|
```
|
|
192
285
|
|
|
193
|
-
The explicit bucket must contain a storage listed in the reference
|
|
194
|
-
|
|
286
|
+
The explicit bucket must contain a storage listed in the reference and have a
|
|
287
|
+
configured region on the server. Deploy `GetReadCredentialsV2` before upgrading
|
|
288
|
+
clients that use explicit buckets; older servers return `UNIMPLEMENTED` and the
|
|
289
|
+
SDK does not fall back to a potentially incorrect legacy endpoint.
|
|
195
290
|
|
|
196
291
|
## OSS
|
|
197
292
|
|
|
@@ -206,20 +301,67 @@ with ManagementClient(address=os.environ["MANAGEMENT_ADDRESS"]) as management:
|
|
|
206
301
|
source.download_file(os.environ["SOURCE_OBJECT_KEY"], "/tmp/input.h5")
|
|
207
302
|
|
|
208
303
|
# Convert the input into /tmp/output.td before uploading.
|
|
209
|
-
with OSSClient(management, permission="upload", data_type=int(os.environ["UPLOAD_DATA_TYPE"])
|
|
304
|
+
with OSSClient(management, permission="upload", data_type=int(os.environ["UPLOAD_DATA_TYPE"]),
|
|
305
|
+
data_kind="CONVERTED") as destination:
|
|
210
306
|
key = destination.object_prefix + "recording/output.td"
|
|
211
307
|
destination.upload_file(key, "/tmp/output.td")
|
|
212
308
|
print(destination.bucket, key) # Use these values when registering metadata.
|
|
213
309
|
```
|
|
214
310
|
|
|
215
|
-
- Read access covers the requested bucket, subject to server permissions.
|
|
216
|
-
The server chooses the upload bucket and prefix based on the requested data type
|
|
311
|
+
- Read access uses V2 and covers the requested bucket, subject to server permissions.
|
|
312
|
+
The server chooses the upload bucket and prefix based on the requested data type
|
|
313
|
+
and kind. Supplying `data_kind` selects V2 and requires `data_type`; omit
|
|
314
|
+
`data_kind` to retain the legacy upload API. Read clients reject both arguments.
|
|
315
|
+
- Reads and V2 uploads use the public endpoint by default. Set `use_internal_endpoint=True`
|
|
316
|
+
on `OSSClient` to select internal networking. Legacy uploads reject this option.
|
|
217
317
|
- Credentials refresh automatically. Expired tokens trigger one refresh and retry.
|
|
218
318
|
- Uploads reject overwrites by default; set `forbid_overwrite=False` to replace.
|
|
219
319
|
Downloads replace the local file only on success; its parent directory must exist.
|
|
220
320
|
- Uploads use a single PutObject request ([maximum 5 GiB](https://www.alibabacloud.com/help/en/oss/developer-reference/upload-files-using-oss-sdk-for-python-v2/)); multipart upload is not supported.
|
|
221
|
-
- Supply `region=` for custom upload endpoints
|
|
222
|
-
|
|
321
|
+
- Supply `region=` for legacy custom upload endpoints; V2 returns its own region.
|
|
322
|
+
Deploy the V2 credential APIs and configure bucket regions before using reads
|
|
323
|
+
or the upload `data_kind` option.
|
|
324
|
+
|
|
325
|
+
## Pre-annotation
|
|
326
|
+
|
|
327
|
+
`current_data_py.labeling.LabelingClient` wraps the pre-annotation RPCs
|
|
328
|
+
`CreateTask`, `ListTasks`, `ListItems`, `WriteResult`, and `UpdatePreAnnotation` with
|
|
329
|
+
ManagementClient's TLS, package default address, caller-owned channel, per-call
|
|
330
|
+
`timeout` (30 seconds by default, `None` disables it), and `metadata` conventions.
|
|
331
|
+
Errors propagate without retries. Protobuf types are available through
|
|
332
|
+
`service_pb2`, `task_pb2`, and `DataSelection` in the same module.
|
|
333
|
+
|
|
334
|
+
```python
|
|
335
|
+
from current_data_py.labeling import LabelingClient, service_pb2, task_pb2
|
|
336
|
+
|
|
337
|
+
with LabelingClient() as labeling:
|
|
338
|
+
task = labeling.create_task(
|
|
339
|
+
"template-id", name="Pre-annotation",
|
|
340
|
+
input_sources=[task_pb2.InputSource(td_file=task_pb2.TdFileSource(data_id="data-id"))],
|
|
341
|
+
start_pre_annotation=True,
|
|
342
|
+
)
|
|
343
|
+
for item in labeling.list_items(task.task_id):
|
|
344
|
+
# Match question keys and option keys to the existing built-in template.
|
|
345
|
+
labeling.write_result(item.item_id, task_pb2.Result(question_results=[
|
|
346
|
+
task_pb2.QuestionResult(
|
|
347
|
+
question_key="quality",
|
|
348
|
+
choice=task_pb2.ChoiceResult(selected_option_keys=["ok"]),
|
|
349
|
+
),
|
|
350
|
+
]))
|
|
351
|
+
# On inference failure instead: write_result(item.item_id, pre_annotation_error="reason")
|
|
352
|
+
labeling.update_pre_annotation(
|
|
353
|
+
task.task_id, service_pb2.UpdatePreAnnotationRequest.ACTION_OPEN_HUMAN_REVIEW,
|
|
354
|
+
)
|
|
355
|
+
```
|
|
356
|
+
|
|
357
|
+
Alternatively, pass `data_selection=DataSelection(...)` when creating a task.
|
|
358
|
+
Use `ACTION_ACCEPT_ALL_AND_COMPLETE` to accept all pre-annotations as final labels.
|
|
359
|
+
Use an existing built-in template; custom templates do not support pre-annotation.
|
|
360
|
+
The server validates inputs and task transitions.
|
|
361
|
+
|
|
362
|
+
`list_tasks` and `list_items` return repeated protobuf collections.
|
|
363
|
+
`create_task` and `update_pre_annotation` return full protobuf responses
|
|
364
|
+
(including item count and task ID, or status). `write_result` returns `None`.
|
|
223
365
|
|
|
224
366
|
## 打包
|
|
225
367
|
|
|
@@ -240,6 +382,7 @@ python -m pip install --force-reinstall dist/*.whl
|
|
|
240
382
|
python -I tests/check_package.py
|
|
241
383
|
python -I tests/check_environment.py
|
|
242
384
|
python -I tests/check_management.py
|
|
385
|
+
python -I tests/check_labeling.py
|
|
243
386
|
python -I tests/check_remote.py
|
|
244
387
|
python -I tests/check_oss.py
|
|
245
388
|
```
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: current-data-py
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.11
|
|
4
4
|
Summary: Current Data protobuf messages and schema metadata
|
|
5
5
|
Requires-Python: >=3.9
|
|
6
6
|
Description-Content-Type: text/markdown
|
|
@@ -83,6 +83,97 @@ SDK. Registration only saves metadata; it does not upload files. Credential
|
|
|
83
83
|
calls return fresh credentials without caching. Calls accept `timeout=` and
|
|
84
84
|
`metadata=` for gRPC headers.
|
|
85
85
|
|
|
86
|
+
For uploads that distinguish raw and converted data, use V2 with both arguments:
|
|
87
|
+
|
|
88
|
+
```python
|
|
89
|
+
with ManagementClient(address=os.environ["MANAGEMENT_ADDRESS"]) as client:
|
|
90
|
+
raw = client.get_upload_credentials_v2(data_type="YAM", data_kind="RAW")
|
|
91
|
+
converted = client.get_upload_credentials_v2(
|
|
92
|
+
data_type="YAM", data_kind="CONVERTED", use_internal_endpoint=True,
|
|
93
|
+
)
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
V2 accepts `EGOSUITE`, `YAM`, and `CURRENT_V2` data types and `RAW` or `CONVERTED`
|
|
97
|
+
data kinds. Each argument also accepts its full enum name (`DATA_TYPE_YAM`,
|
|
98
|
+
`DATA_KIND_RAW`) or integer value. `DataKind` is exported from
|
|
99
|
+
`current_data_py.management`; data type constants are in `metadata_pb2`.
|
|
100
|
+
Unspecified or unknown enum values are rejected. `RAW` is not a V2 data type.
|
|
101
|
+
The server must have the selected environment/type/kind destination configured;
|
|
102
|
+
unfilled placeholders return `FAILED_PRECONDITION`. V2 requires a deployed
|
|
103
|
+
`GetUploadCredentialsV2` RPC and never falls back to the old RPC on failure.
|
|
104
|
+
The existing `get_upload_credentials()` API and its defaults remain unchanged.
|
|
105
|
+
V2 responses include the selected destination's `oss_endpoint` and `region`,
|
|
106
|
+
along with its bucket and prefix; raw and converted destinations may use different
|
|
107
|
+
OSS regions. The server derives the standard Alibaba Cloud endpoint from `region`:
|
|
108
|
+
`https://oss-<region>.aliyuncs.com` by default, or
|
|
109
|
+
`https://oss-<region>-internal.aliyuncs.com` with `use_internal_endpoint=True`.
|
|
110
|
+
Use the internal endpoint from Alibaba Cloud resources in the same region.
|
|
111
|
+
See the [Alibaba Cloud endpoint reference](https://www.alibabacloud.com/help/en/oss/user-guide/regions-and-endpoints).
|
|
112
|
+
`OSSClient` accepts the same option for V2 uploads and retains it when refreshing
|
|
113
|
+
credentials. It uses the returned endpoint and region for connections and signing.
|
|
114
|
+
An explicit `region=` must match the returned region. If the endpoint or
|
|
115
|
+
region changes during credential refresh, create a new `OSSClient`.
|
|
116
|
+
|
|
117
|
+
For reads in the bucket's own region, use the new read V2 RPC:
|
|
118
|
+
|
|
119
|
+
```python
|
|
120
|
+
with ManagementClient(address=os.environ["MANAGEMENT_ADDRESS"]) as client:
|
|
121
|
+
read = client.get_read_credentials_v2(
|
|
122
|
+
bucket=os.environ["READ_BUCKET"], use_internal_endpoint=False,
|
|
123
|
+
)
|
|
124
|
+
```
|
|
125
|
+
|
|
126
|
+
The bucket is required and must appear in the service's environment-specific
|
|
127
|
+
bucket-to-region map, shared with V2 uploads. Reads grant whole-bucket `GetObject`
|
|
128
|
+
access bounded by the read RAM role. Public networking is the default; set
|
|
129
|
+
`use_internal_endpoint=True` for internal networking. Unconfigured buckets return
|
|
130
|
+
`FAILED_PRECONDITION`; empty or malformed buckets return `INVALID_ARGUMENT`.
|
|
131
|
+
The existing `get_read_credentials()` API keeps its configured default endpoint,
|
|
132
|
+
region and scope. V2 calls never fall back to it on failure.
|
|
133
|
+
|
|
134
|
+
### Multi-cloud credentials (V3)
|
|
135
|
+
|
|
136
|
+
Use V3 to explicitly choose OSS or native TOS. `StorageProvider` and
|
|
137
|
+
`StorageCredentials` are exported from `current_data_py.management`.
|
|
138
|
+
|
|
139
|
+
```python
|
|
140
|
+
from current_data_py.management import ManagementClient
|
|
141
|
+
|
|
142
|
+
with ManagementClient() as client:
|
|
143
|
+
upload = client.get_upload_credentials_v3(
|
|
144
|
+
provider="TOS", data_type="YAM", data_kind="RAW",
|
|
145
|
+
use_internal_endpoint=False,
|
|
146
|
+
)
|
|
147
|
+
read = client.get_read_credentials_v3(
|
|
148
|
+
provider="TOS", bucket=upload.bucket,
|
|
149
|
+
use_internal_endpoint=False,
|
|
150
|
+
)
|
|
151
|
+
print(upload.provider, upload.bucket, upload.endpoint, upload.region)
|
|
152
|
+
```
|
|
153
|
+
|
|
154
|
+
`provider` is required and accepts `OSS`/`TOS`, full enum names such as
|
|
155
|
+
`STORAGE_PROVIDER_TOS`, or enum integer values. Unspecified/unknown providers
|
|
156
|
+
are rejected. Uploads require the same explicit type/kind as V2; reads require
|
|
157
|
+
an explicitly configured bucket on the selected provider. OSS V3 uses the same
|
|
158
|
+
routes and roles as OSS V2. TOS configuration and IAM preparation are described
|
|
159
|
+
in the [service deployment guide](../management_service/README.md#tos-preparation-for-v3).
|
|
160
|
+
|
|
161
|
+
Both calls return `provider`, `endpoint`, `region`, `bucket`, `object_prefix`,
|
|
162
|
+
`access_key_id`, `access_key_secret`, `security_token`, and `expires_at`.
|
|
163
|
+
`object_prefix` is empty for whole-bucket reads. Use `endpoint` instead of V2's
|
|
164
|
+
`oss_endpoint`. TOS public endpoints use `tos-<region>.volces.com`; internal
|
|
165
|
+
endpoints use `tos-<region>.ivolces.com`. Internal networking requires access to
|
|
166
|
+
the destination cloud and region's private network.
|
|
167
|
+
|
|
168
|
+
V3 calls accept `timeout` and `metadata`, do not cache credentials, and never
|
|
169
|
+
retry or fall back to earlier RPC versions. Old servers return `UNIMPLEMENTED`;
|
|
170
|
+
unconfigured destinations return `FAILED_PRECONDITION`. A response whose
|
|
171
|
+
provider differs from the request raises `ValueError`.
|
|
172
|
+
|
|
173
|
+
These methods only fetch credentials. Pass TOS credentials to a native TOS
|
|
174
|
+
client and refresh them before expiration. `OSSClient`, `DataClient`, metadata
|
|
175
|
+
registration and signed URLs have not been extended to TOS by this change.
|
|
176
|
+
|
|
86
177
|
## Remote TD reads
|
|
87
178
|
|
|
88
179
|
### Payload decoding
|
|
@@ -183,10 +274,12 @@ to each call.
|
|
|
183
274
|
|
|
184
275
|
`DataClient.open_td()` accepts an optional `bucket` argument. Without it, reads
|
|
185
276
|
use the management service's `OSS_READ_BUCKET` and `OSS_READ_PREFIX` defaults.
|
|
186
|
-
With it, the SDK
|
|
187
|
-
RAM role, and selects the first matching
|
|
188
|
-
bucket does not inherit the default prefix.
|
|
189
|
-
|
|
277
|
+
With it, the SDK uses `GetReadCredentialsV2` for whole-bucket credentials in the
|
|
278
|
+
bucket's region, limited by the server's RAM role, and selects the first matching
|
|
279
|
+
storage in the reference. An explicit bucket does not inherit the default prefix.
|
|
280
|
+
Set `use_internal_endpoint=True` with an explicit bucket to use internal networking.
|
|
281
|
+
Credentials are cached separately for each `(bucket, use_internal_endpoint)` pair,
|
|
282
|
+
apart from the legacy default scope, and refreshed automatically.
|
|
190
283
|
|
|
191
284
|
```python
|
|
192
285
|
import os
|
|
@@ -204,8 +297,10 @@ with DataClient(address=os.environ["MANAGEMENT_ADDRESS"]) as client:
|
|
|
204
297
|
print(reader.summary())
|
|
205
298
|
```
|
|
206
299
|
|
|
207
|
-
The explicit bucket must contain a storage listed in the reference
|
|
208
|
-
|
|
300
|
+
The explicit bucket must contain a storage listed in the reference and have a
|
|
301
|
+
configured region on the server. Deploy `GetReadCredentialsV2` before upgrading
|
|
302
|
+
clients that use explicit buckets; older servers return `UNIMPLEMENTED` and the
|
|
303
|
+
SDK does not fall back to a potentially incorrect legacy endpoint.
|
|
209
304
|
|
|
210
305
|
## OSS
|
|
211
306
|
|
|
@@ -220,20 +315,67 @@ with ManagementClient(address=os.environ["MANAGEMENT_ADDRESS"]) as management:
|
|
|
220
315
|
source.download_file(os.environ["SOURCE_OBJECT_KEY"], "/tmp/input.h5")
|
|
221
316
|
|
|
222
317
|
# Convert the input into /tmp/output.td before uploading.
|
|
223
|
-
with OSSClient(management, permission="upload", data_type=int(os.environ["UPLOAD_DATA_TYPE"])
|
|
318
|
+
with OSSClient(management, permission="upload", data_type=int(os.environ["UPLOAD_DATA_TYPE"]),
|
|
319
|
+
data_kind="CONVERTED") as destination:
|
|
224
320
|
key = destination.object_prefix + "recording/output.td"
|
|
225
321
|
destination.upload_file(key, "/tmp/output.td")
|
|
226
322
|
print(destination.bucket, key) # Use these values when registering metadata.
|
|
227
323
|
```
|
|
228
324
|
|
|
229
|
-
- Read access covers the requested bucket, subject to server permissions.
|
|
230
|
-
The server chooses the upload bucket and prefix based on the requested data type
|
|
325
|
+
- Read access uses V2 and covers the requested bucket, subject to server permissions.
|
|
326
|
+
The server chooses the upload bucket and prefix based on the requested data type
|
|
327
|
+
and kind. Supplying `data_kind` selects V2 and requires `data_type`; omit
|
|
328
|
+
`data_kind` to retain the legacy upload API. Read clients reject both arguments.
|
|
329
|
+
- Reads and V2 uploads use the public endpoint by default. Set `use_internal_endpoint=True`
|
|
330
|
+
on `OSSClient` to select internal networking. Legacy uploads reject this option.
|
|
231
331
|
- Credentials refresh automatically. Expired tokens trigger one refresh and retry.
|
|
232
332
|
- Uploads reject overwrites by default; set `forbid_overwrite=False` to replace.
|
|
233
333
|
Downloads replace the local file only on success; its parent directory must exist.
|
|
234
334
|
- Uploads use a single PutObject request ([maximum 5 GiB](https://www.alibabacloud.com/help/en/oss/developer-reference/upload-files-using-oss-sdk-for-python-v2/)); multipart upload is not supported.
|
|
235
|
-
- Supply `region=` for custom upload endpoints
|
|
236
|
-
|
|
335
|
+
- Supply `region=` for legacy custom upload endpoints; V2 returns its own region.
|
|
336
|
+
Deploy the V2 credential APIs and configure bucket regions before using reads
|
|
337
|
+
or the upload `data_kind` option.
|
|
338
|
+
|
|
339
|
+
## Pre-annotation
|
|
340
|
+
|
|
341
|
+
`current_data_py.labeling.LabelingClient` wraps the pre-annotation RPCs
|
|
342
|
+
`CreateTask`, `ListTasks`, `ListItems`, `WriteResult`, and `UpdatePreAnnotation` with
|
|
343
|
+
ManagementClient's TLS, package default address, caller-owned channel, per-call
|
|
344
|
+
`timeout` (30 seconds by default, `None` disables it), and `metadata` conventions.
|
|
345
|
+
Errors propagate without retries. Protobuf types are available through
|
|
346
|
+
`service_pb2`, `task_pb2`, and `DataSelection` in the same module.
|
|
347
|
+
|
|
348
|
+
```python
|
|
349
|
+
from current_data_py.labeling import LabelingClient, service_pb2, task_pb2
|
|
350
|
+
|
|
351
|
+
with LabelingClient() as labeling:
|
|
352
|
+
task = labeling.create_task(
|
|
353
|
+
"template-id", name="Pre-annotation",
|
|
354
|
+
input_sources=[task_pb2.InputSource(td_file=task_pb2.TdFileSource(data_id="data-id"))],
|
|
355
|
+
start_pre_annotation=True,
|
|
356
|
+
)
|
|
357
|
+
for item in labeling.list_items(task.task_id):
|
|
358
|
+
# Match question keys and option keys to the existing built-in template.
|
|
359
|
+
labeling.write_result(item.item_id, task_pb2.Result(question_results=[
|
|
360
|
+
task_pb2.QuestionResult(
|
|
361
|
+
question_key="quality",
|
|
362
|
+
choice=task_pb2.ChoiceResult(selected_option_keys=["ok"]),
|
|
363
|
+
),
|
|
364
|
+
]))
|
|
365
|
+
# On inference failure instead: write_result(item.item_id, pre_annotation_error="reason")
|
|
366
|
+
labeling.update_pre_annotation(
|
|
367
|
+
task.task_id, service_pb2.UpdatePreAnnotationRequest.ACTION_OPEN_HUMAN_REVIEW,
|
|
368
|
+
)
|
|
369
|
+
```
|
|
370
|
+
|
|
371
|
+
Alternatively, pass `data_selection=DataSelection(...)` when creating a task.
|
|
372
|
+
Use `ACTION_ACCEPT_ALL_AND_COMPLETE` to accept all pre-annotations as final labels.
|
|
373
|
+
Use an existing built-in template; custom templates do not support pre-annotation.
|
|
374
|
+
The server validates inputs and task transitions.
|
|
375
|
+
|
|
376
|
+
`list_tasks` and `list_items` return repeated protobuf collections.
|
|
377
|
+
`create_task` and `update_pre_annotation` return full protobuf responses
|
|
378
|
+
(including item count and task ID, or status). `write_result` returns `None`.
|
|
237
379
|
|
|
238
380
|
## 打包
|
|
239
381
|
|
|
@@ -254,6 +396,7 @@ python -m pip install --force-reinstall dist/*.whl
|
|
|
254
396
|
python -I tests/check_package.py
|
|
255
397
|
python -I tests/check_environment.py
|
|
256
398
|
python -I tests/check_management.py
|
|
399
|
+
python -I tests/check_labeling.py
|
|
257
400
|
python -I tests/check_remote.py
|
|
258
401
|
python -I tests/check_oss.py
|
|
259
402
|
```
|