current-data-py 0.1.10__tar.gz → 0.1.12__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (95) hide show
  1. {current_data_py-0.1.10 → current_data_py-0.1.12}/MANIFEST.in +1 -0
  2. {current_data_py-0.1.10 → current_data_py-0.1.12}/PKG-INFO +78 -13
  3. current_data_py-0.1.10/current_data_py.egg-info/PKG-INFO → current_data_py-0.1.12/README.md +74 -23
  4. {current_data_py-0.1.10 → current_data_py-0.1.12}/__init__.py +0 -0
  5. current_data_py-0.1.10/README.md → current_data_py-0.1.12/current_data_py.egg-info/PKG-INFO +88 -9
  6. {current_data_py-0.1.10 → current_data_py-0.1.12}/current_data_py.egg-info/SOURCES.txt +7 -0
  7. {current_data_py-0.1.10 → current_data_py-0.1.12}/current_data_py.egg-info/dependency_links.txt +0 -0
  8. {current_data_py-0.1.10 → current_data_py-0.1.12}/current_data_py.egg-info/requires.txt +1 -2
  9. {current_data_py-0.1.10 → current_data_py-0.1.12}/current_data_py.egg-info/top_level.txt +0 -0
  10. {current_data_py-0.1.10 → current_data_py-0.1.12}/decoding.py +3 -6
  11. {current_data_py-0.1.10 → current_data_py-0.1.12}/examples/__init__.py +0 -0
  12. {current_data_py-0.1.10 → current_data_py-0.1.12}/examples/list_schemas.py +0 -0
  13. {current_data_py-0.1.10 → current_data_py-0.1.12}/examples/read_remote_td.py +0 -0
  14. {current_data_py-0.1.10 → current_data_py-0.1.12}/examples/read_td.py +0 -0
  15. {current_data_py-0.1.10 → current_data_py-0.1.12}/examples/write_td.py +0 -0
  16. current_data_py-0.1.12/labeling.py +102 -0
  17. {current_data_py-0.1.10 → current_data_py-0.1.12}/management.py +3 -3
  18. {current_data_py-0.1.10 → current_data_py-0.1.12}/oss.py +0 -0
  19. current_data_py-0.1.12/proto_gen/python/labeling/service/service_pb2.py +82 -0
  20. current_data_py-0.1.12/proto_gen/python/labeling/service/service_pb2_grpc.py +433 -0
  21. current_data_py-0.1.12/proto_gen/python/labeling/task/task_pb2.py +50 -0
  22. current_data_py-0.1.12/proto_gen/python/labeling/template/template_pb2.py +40 -0
  23. current_data_py-0.1.12/proto_gen/python/management/metadata/metadata_pb2.py +64 -0
  24. current_data_py-0.1.12/proto_gen/python/management/selection/selection_pb2.py +27 -0
  25. current_data_py-0.1.12/proto_gen/python/management/service/service_pb2.py +80 -0
  26. {current_data_py-0.1.10 → current_data_py-0.1.12}/proto_gen/python/management/service/service_pb2_grpc.py +33 -0
  27. {current_data_py-0.1.10 → current_data_py-0.1.12}/proto_gen/python/msg/builtin_interfaces/duration_pb2.py +0 -0
  28. {current_data_py-0.1.10 → current_data_py-0.1.12}/proto_gen/python/msg/builtin_interfaces/time_pb2.py +0 -0
  29. {current_data_py-0.1.10 → current_data_py-0.1.12}/proto_gen/python/msg/custom_msgs/joint_current_pb2.py +0 -0
  30. {current_data_py-0.1.10 → current_data_py-0.1.12}/proto_gen/python/msg/custom_msgs/pose_rpy_pb2.py +0 -0
  31. {current_data_py-0.1.10 → current_data_py-0.1.12}/proto_gen/python/msg/custom_msgs/slam_quality_pb2.py +0 -0
  32. {current_data_py-0.1.10 → current_data_py-0.1.12}/proto_gen/python/msg/custom_msgs/sonic_frame_pb2.py +0 -0
  33. {current_data_py-0.1.10 → current_data_py-0.1.12}/proto_gen/python/msg/custom_msgs/target_quality_pb2.py +0 -0
  34. {current_data_py-0.1.10 → current_data_py-0.1.12}/proto_gen/python/msg/foxglove/compressed_audio_pb2.py +0 -0
  35. {current_data_py-0.1.10 → current_data_py-0.1.12}/proto_gen/python/msg/foxglove/compressed_video_pb2.py +0 -0
  36. {current_data_py-0.1.10 → current_data_py-0.1.12}/proto_gen/python/msg/geometry_msgs/point32_pb2.py +0 -0
  37. {current_data_py-0.1.10 → current_data_py-0.1.12}/proto_gen/python/msg/geometry_msgs/point_pb2.py +0 -0
  38. {current_data_py-0.1.10 → current_data_py-0.1.12}/proto_gen/python/msg/geometry_msgs/pose_pb2.py +0 -0
  39. {current_data_py-0.1.10 → current_data_py-0.1.12}/proto_gen/python/msg/geometry_msgs/pose_stamped_pb2.py +0 -0
  40. {current_data_py-0.1.10 → current_data_py-0.1.12}/proto_gen/python/msg/geometry_msgs/pose_with_covariance_pb2.py +0 -0
  41. {current_data_py-0.1.10 → current_data_py-0.1.12}/proto_gen/python/msg/geometry_msgs/pose_with_covariance_stamped_pb2.py +0 -0
  42. {current_data_py-0.1.10 → current_data_py-0.1.12}/proto_gen/python/msg/geometry_msgs/quaternion_pb2.py +0 -0
  43. {current_data_py-0.1.10 → current_data_py-0.1.12}/proto_gen/python/msg/geometry_msgs/transform_pb2.py +0 -0
  44. {current_data_py-0.1.10 → current_data_py-0.1.12}/proto_gen/python/msg/geometry_msgs/transform_stamped_pb2.py +0 -0
  45. {current_data_py-0.1.10 → current_data_py-0.1.12}/proto_gen/python/msg/geometry_msgs/twist_pb2.py +0 -0
  46. {current_data_py-0.1.10 → current_data_py-0.1.12}/proto_gen/python/msg/geometry_msgs/twist_stamped_pb2.py +0 -0
  47. {current_data_py-0.1.10 → current_data_py-0.1.12}/proto_gen/python/msg/geometry_msgs/twist_with_covariance_pb2.py +0 -0
  48. {current_data_py-0.1.10 → current_data_py-0.1.12}/proto_gen/python/msg/geometry_msgs/twist_with_covariance_stamped_pb2.py +0 -0
  49. {current_data_py-0.1.10 → current_data_py-0.1.12}/proto_gen/python/msg/geometry_msgs/vector3_pb2.py +0 -0
  50. {current_data_py-0.1.10 → current_data_py-0.1.12}/proto_gen/python/msg/geometry_msgs/wrench_pb2.py +0 -0
  51. {current_data_py-0.1.10 → current_data_py-0.1.12}/proto_gen/python/msg/geometry_msgs/wrench_stamped_pb2.py +0 -0
  52. {current_data_py-0.1.10 → current_data_py-0.1.12}/proto_gen/python/msg/nav_msgs/map_meta_data_pb2.py +0 -0
  53. {current_data_py-0.1.10 → current_data_py-0.1.12}/proto_gen/python/msg/nav_msgs/occupancy_grid_pb2.py +0 -0
  54. {current_data_py-0.1.10 → current_data_py-0.1.12}/proto_gen/python/msg/nav_msgs/odometry_pb2.py +0 -0
  55. {current_data_py-0.1.10 → current_data_py-0.1.12}/proto_gen/python/msg/nav_msgs/path_pb2.py +0 -0
  56. {current_data_py-0.1.10 → current_data_py-0.1.12}/proto_gen/python/msg/sensor_msgs/battery_state_pb2.py +0 -0
  57. {current_data_py-0.1.10 → current_data_py-0.1.12}/proto_gen/python/msg/sensor_msgs/camera_info_pb2.py +0 -0
  58. {current_data_py-0.1.10 → current_data_py-0.1.12}/proto_gen/python/msg/sensor_msgs/compressed_image_pb2.py +0 -0
  59. {current_data_py-0.1.10 → current_data_py-0.1.12}/proto_gen/python/msg/sensor_msgs/image_pb2.py +0 -0
  60. {current_data_py-0.1.10 → current_data_py-0.1.12}/proto_gen/python/msg/sensor_msgs/imu_pb2.py +0 -0
  61. {current_data_py-0.1.10 → current_data_py-0.1.12}/proto_gen/python/msg/sensor_msgs/joint_state_pb2.py +0 -0
  62. {current_data_py-0.1.10 → current_data_py-0.1.12}/proto_gen/python/msg/sensor_msgs/laser_scan_pb2.py +0 -0
  63. {current_data_py-0.1.10 → current_data_py-0.1.12}/proto_gen/python/msg/sensor_msgs/nav_sat_fix_pb2.py +0 -0
  64. {current_data_py-0.1.10 → current_data_py-0.1.12}/proto_gen/python/msg/sensor_msgs/nav_sat_status_pb2.py +0 -0
  65. {current_data_py-0.1.10 → current_data_py-0.1.12}/proto_gen/python/msg/sensor_msgs/point_cloud2_pb2.py +0 -0
  66. {current_data_py-0.1.10 → current_data_py-0.1.12}/proto_gen/python/msg/sensor_msgs/point_field_pb2.py +0 -0
  67. {current_data_py-0.1.10 → current_data_py-0.1.12}/proto_gen/python/msg/sensor_msgs/range_pb2.py +0 -0
  68. {current_data_py-0.1.10 → current_data_py-0.1.12}/proto_gen/python/msg/sensor_msgs/region_of_interest_pb2.py +0 -0
  69. {current_data_py-0.1.10 → current_data_py-0.1.12}/proto_gen/python/msg/std_msgs/bool_pb2.py +0 -0
  70. {current_data_py-0.1.10 → current_data_py-0.1.12}/proto_gen/python/msg/std_msgs/color_rgba_pb2.py +0 -0
  71. {current_data_py-0.1.10 → current_data_py-0.1.12}/proto_gen/python/msg/std_msgs/float32_pb2.py +0 -0
  72. {current_data_py-0.1.10 → current_data_py-0.1.12}/proto_gen/python/msg/std_msgs/float64_pb2.py +0 -0
  73. {current_data_py-0.1.10 → current_data_py-0.1.12}/proto_gen/python/msg/std_msgs/header_pb2.py +0 -0
  74. {current_data_py-0.1.10 → current_data_py-0.1.12}/proto_gen/python/msg/std_msgs/int32_pb2.py +0 -0
  75. {current_data_py-0.1.10 → current_data_py-0.1.12}/proto_gen/python/msg/std_msgs/string_pb2.py +0 -0
  76. {current_data_py-0.1.10 → current_data_py-0.1.12}/proto_gen/python/msg/std_msgs/uint32_pb2.py +0 -0
  77. {current_data_py-0.1.10 → current_data_py-0.1.12}/proto_gen/python/msg/tf2_msgs/tf_message_pb2.py +0 -0
  78. {current_data_py-0.1.10 → current_data_py-0.1.12}/pyproject.toml +4 -4
  79. {current_data_py-0.1.10 → current_data_py-0.1.12}/remote/__init__.py +0 -0
  80. {current_data_py-0.1.10 → current_data_py-0.1.12}/remote/client.py +53 -12
  81. {current_data_py-0.1.10 → current_data_py-0.1.12}/remote/source.py +2 -1
  82. current_data_py-0.1.12/remote/tos_source.py +84 -0
  83. {current_data_py-0.1.10 → current_data_py-0.1.12}/schema/__init__.py +0 -0
  84. {current_data_py-0.1.10 → current_data_py-0.1.12}/schema/message_metadata.py +0 -0
  85. {current_data_py-0.1.10 → current_data_py-0.1.12}/setup.cfg +0 -0
  86. {current_data_py-0.1.10 → current_data_py-0.1.12}/setup.py +1 -1
  87. {current_data_py-0.1.10 → current_data_py-0.1.12}/tests/check_environment.py +0 -0
  88. current_data_py-0.1.12/tests/check_labeling.py +111 -0
  89. {current_data_py-0.1.10 → current_data_py-0.1.12}/tests/check_management.py +2 -2
  90. {current_data_py-0.1.10 → current_data_py-0.1.12}/tests/check_oss.py +0 -0
  91. {current_data_py-0.1.10 → current_data_py-0.1.12}/tests/check_package.py +0 -0
  92. {current_data_py-0.1.10 → current_data_py-0.1.12}/tests/check_remote.py +4 -3
  93. current_data_py-0.1.12/tos.py +170 -0
  94. current_data_py-0.1.10/proto_gen/python/management/metadata/metadata_pb2.py +0 -62
  95. current_data_py-0.1.10/proto_gen/python/management/service/service_pb2.py +0 -83
@@ -3,3 +3,4 @@ include tests/check_environment.py
3
3
  include tests/check_remote.py
4
4
  include tests/check_management.py
5
5
  include tests/check_oss.py
6
+ include tests/check_labeling.py
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: current-data-py
3
- Version: 0.1.10
3
+ Version: 0.1.12
4
4
  Summary: Current Data protobuf messages and schema metadata
5
5
  Requires-Python: >=3.9
6
6
  Description-Content-Type: text/markdown
@@ -8,9 +8,9 @@ Requires-Dist: protobuf>=3.20
8
8
  Requires-Dist: turbodata==0.1.0
9
9
  Requires-Dist: grpcio>=1.51.1
10
10
  Requires-Dist: alibabacloud-oss-v2<2,>=1.4.0
11
- Provides-Extra: decode
12
- Requires-Dist: av>=12; extra == "decode"
13
- Requires-Dist: pillow>=9; extra == "decode"
11
+ Requires-Dist: tos<3,>=2.9
12
+ Requires-Dist: av>=12
13
+ Requires-Dist: pillow>=9
14
14
 
15
15
  # current-data-py
16
16
 
@@ -170,9 +170,8 @@ retry or fall back to earlier RPC versions. Old servers return `UNIMPLEMENTED`;
170
170
  unconfigured destinations return `FAILED_PRECONDITION`. A response whose
171
171
  provider differs from the request raises `ValueError`.
172
172
 
173
- These methods only fetch credentials. Pass TOS credentials to a native TOS
174
- client and refresh them before expiration. `OSSClient`, `DataClient`, metadata
175
- registration and signed URLs have not been extended to TOS by this change.
173
+ These methods only fetch credentials. For file transfers and TD reads with automatic
174
+ credential refresh, see [TOS](#tos). Signed download URLs remain OSS-only.
176
175
 
177
176
  ## Remote TD reads
178
177
 
@@ -180,12 +179,11 @@ registration and signed URLs have not been extended to TOS by this change.
180
179
 
181
180
  > [!IMPORTANT]
182
181
  > **Video decoding requires PyAV, Pillow and the FFmpeg native libraries used by
183
- > PyAV.** `open_td()` enables decoding by default, but a base `current-data-py`
184
- > installation does not install the video dependencies. Install the `[decode]`
185
- > extra in every environment that decodes video, including DataLoader workers.
182
+ > PyAV.** `current-data-py` installs PyAV and Pillow by default. Install the package
183
+ > in every environment that decodes video, including DataLoader workers.
186
184
 
187
185
  ```bash
188
- python -m pip install --only-binary=av,pillow 'current-data-py[decode]'
186
+ python -m pip install --only-binary=av,pillow current-data-py
189
187
  ```
190
188
 
191
189
  - **Prebuilt wheels:** PyAV wheels bundle FFmpeg libraries on supported platforms;
@@ -272,8 +270,9 @@ to each call.
272
270
 
273
271
  ### Storage and credentials
274
272
 
275
- `DataClient.open_td()` accepts an optional `bucket` argument. Without it, reads
276
- use the management service's `OSS_READ_BUCKET` and `OSS_READ_PREFIX` defaults.
273
+ `DataClient.open_td()` accepts optional `bucket` and `provider` arguments. OSS
274
+ references without either selection retain the service's `OSS_READ_BUCKET` and
275
+ `OSS_READ_PREFIX` defaults. See [TOS](#tos) for provider selection.
277
276
  With it, the SDK uses `GetReadCredentialsV2` for whole-bucket credentials in the
278
277
  bucket's region, limited by the server's RAM role, and selects the first matching
279
278
  storage in the reference. An explicit bucket does not inherit the default prefix.
@@ -336,6 +335,69 @@ with ManagementClient(address=os.environ["MANAGEMENT_ADDRESS"]) as management:
336
335
  Deploy the V2 credential APIs and configure bucket regions before using reads
337
336
  or the upload `data_kind` option.
338
337
 
338
+ ## TOS
339
+
340
+ `TOSClient` obtains and refreshes V3 credentials from Management. Keep
341
+ `ManagementClient` open during transfers.
342
+
343
+ ```python
344
+ import os
345
+ from current_data_py.management import ManagementClient
346
+ from current_data_py.tos import TOSClient
347
+
348
+ with ManagementClient() as management:
349
+ with TOSClient(management, permission="read", bucket=os.environ["TOS_READ_BUCKET"],
350
+ endpoint="https://tos-accelerate.volces.com") as source:
351
+ source.download_file(os.environ["SOURCE_OBJECT_KEY"], "/tmp/input.h5")
352
+ ```
353
+
354
+ For uploads, use `permission="upload", data_type="YAM", data_kind="CONVERTED"`;
355
+ register the returned bucket and uploaded object key in `Metadata.storages`. To read a
356
+ TD file, use `DataClient.open_td(reference, provider="tos", bucket=bucket)`.
357
+ `endpoint=` changes the network endpoint without changing the signing region;
358
+ it cannot be combined with `use_internal_endpoint=True`.
359
+
360
+ ## Pre-annotation
361
+
362
+ `current_data_py.labeling.LabelingClient` wraps the pre-annotation RPCs
363
+ `CreateTask`, `ListTasks`, `ListItems`, `WriteResult`, and `UpdatePreAnnotation` with
364
+ ManagementClient's TLS, package default address, caller-owned channel, per-call
365
+ `timeout` (30 seconds by default, `None` disables it), and `metadata` conventions.
366
+ Errors propagate without retries. Protobuf types are available through
367
+ `service_pb2`, `task_pb2`, and `DataSelection` in the same module.
368
+
369
+ ```python
370
+ from current_data_py.labeling import LabelingClient, service_pb2, task_pb2
371
+
372
+ with LabelingClient() as labeling:
373
+ task = labeling.create_task(
374
+ "template-id", name="Pre-annotation",
375
+ input_sources=[task_pb2.InputSource(td_file=task_pb2.TdFileSource(data_id="data-id"))],
376
+ start_pre_annotation=True,
377
+ )
378
+ for item in labeling.list_items(task.task_id):
379
+ # Match question keys and option keys to the existing built-in template.
380
+ labeling.write_result(item.item_id, task_pb2.Result(question_results=[
381
+ task_pb2.QuestionResult(
382
+ question_key="quality",
383
+ choice=task_pb2.ChoiceResult(selected_option_keys=["ok"]),
384
+ ),
385
+ ]))
386
+ # On inference failure instead: write_result(item.item_id, pre_annotation_error="reason")
387
+ labeling.update_pre_annotation(
388
+ task.task_id, service_pb2.UpdatePreAnnotationRequest.ACTION_OPEN_HUMAN_REVIEW,
389
+ )
390
+ ```
391
+
392
+ Alternatively, pass `data_selection=DataSelection(...)` when creating a task.
393
+ Use `ACTION_ACCEPT_ALL_AND_COMPLETE` to accept all pre-annotations as final labels.
394
+ Use an existing built-in template; custom templates do not support pre-annotation.
395
+ The server validates inputs and task transitions.
396
+
397
+ `list_tasks` and `list_items` return repeated protobuf collections.
398
+ `create_task` and `update_pre_annotation` return full protobuf responses
399
+ (including item count and task ID, or status). `write_result` returns `None`.
400
+
339
401
  ## 打包
340
402
 
341
403
  ```bash
@@ -355,8 +417,11 @@ python -m pip install --force-reinstall dist/*.whl
355
417
  python -I tests/check_package.py
356
418
  python -I tests/check_environment.py
357
419
  python -I tests/check_management.py
420
+ python -I tests/check_labeling.py
358
421
  python -I tests/check_remote.py
359
422
  python -I tests/check_oss.py
423
+ # With the tos extra installed:
424
+ python -I tests/check_tos.py
360
425
  ```
361
426
 
362
427
  ## 推送
@@ -1,17 +1,3 @@
1
- Metadata-Version: 2.4
2
- Name: current-data-py
3
- Version: 0.1.10
4
- Summary: Current Data protobuf messages and schema metadata
5
- Requires-Python: >=3.9
6
- Description-Content-Type: text/markdown
7
- Requires-Dist: protobuf>=3.20
8
- Requires-Dist: turbodata==0.1.0
9
- Requires-Dist: grpcio>=1.51.1
10
- Requires-Dist: alibabacloud-oss-v2<2,>=1.4.0
11
- Provides-Extra: decode
12
- Requires-Dist: av>=12; extra == "decode"
13
- Requires-Dist: pillow>=9; extra == "decode"
14
-
15
1
  # current-data-py
16
2
 
17
3
  ## Environment
@@ -170,9 +156,8 @@ retry or fall back to earlier RPC versions. Old servers return `UNIMPLEMENTED`;
170
156
  unconfigured destinations return `FAILED_PRECONDITION`. A response whose
171
157
  provider differs from the request raises `ValueError`.
172
158
 
173
- These methods only fetch credentials. Pass TOS credentials to a native TOS
174
- client and refresh them before expiration. `OSSClient`, `DataClient`, metadata
175
- registration and signed URLs have not been extended to TOS by this change.
159
+ These methods only fetch credentials. For file transfers and TD reads with automatic
160
+ credential refresh, see [TOS](#tos). Signed download URLs remain OSS-only.
176
161
 
177
162
  ## Remote TD reads
178
163
 
@@ -180,12 +165,11 @@ registration and signed URLs have not been extended to TOS by this change.
180
165
 
181
166
  > [!IMPORTANT]
182
167
  > **Video decoding requires PyAV, Pillow and the FFmpeg native libraries used by
183
- > PyAV.** `open_td()` enables decoding by default, but a base `current-data-py`
184
- > installation does not install the video dependencies. Install the `[decode]`
185
- > extra in every environment that decodes video, including DataLoader workers.
168
+ > PyAV.** `current-data-py` installs PyAV and Pillow by default. Install the package
169
+ > in every environment that decodes video, including DataLoader workers.
186
170
 
187
171
  ```bash
188
- python -m pip install --only-binary=av,pillow 'current-data-py[decode]'
172
+ python -m pip install --only-binary=av,pillow current-data-py
189
173
  ```
190
174
 
191
175
  - **Prebuilt wheels:** PyAV wheels bundle FFmpeg libraries on supported platforms;
@@ -272,8 +256,9 @@ to each call.
272
256
 
273
257
  ### Storage and credentials
274
258
 
275
- `DataClient.open_td()` accepts an optional `bucket` argument. Without it, reads
276
- use the management service's `OSS_READ_BUCKET` and `OSS_READ_PREFIX` defaults.
259
+ `DataClient.open_td()` accepts optional `bucket` and `provider` arguments. OSS
260
+ references without either selection retain the service's `OSS_READ_BUCKET` and
261
+ `OSS_READ_PREFIX` defaults. See [TOS](#tos) for provider selection.
277
262
  With it, the SDK uses `GetReadCredentialsV2` for whole-bucket credentials in the
278
263
  bucket's region, limited by the server's RAM role, and selects the first matching
279
264
  storage in the reference. An explicit bucket does not inherit the default prefix.
@@ -336,6 +321,69 @@ with ManagementClient(address=os.environ["MANAGEMENT_ADDRESS"]) as management:
336
321
  Deploy the V2 credential APIs and configure bucket regions before using reads
337
322
  or the upload `data_kind` option.
338
323
 
324
+ ## TOS
325
+
326
+ `TOSClient` obtains and refreshes V3 credentials from Management. Keep
327
+ `ManagementClient` open during transfers.
328
+
329
+ ```python
330
+ import os
331
+ from current_data_py.management import ManagementClient
332
+ from current_data_py.tos import TOSClient
333
+
334
+ with ManagementClient() as management:
335
+ with TOSClient(management, permission="read", bucket=os.environ["TOS_READ_BUCKET"],
336
+ endpoint="https://tos-accelerate.volces.com") as source:
337
+ source.download_file(os.environ["SOURCE_OBJECT_KEY"], "/tmp/input.h5")
338
+ ```
339
+
340
+ For uploads, use `permission="upload", data_type="YAM", data_kind="CONVERTED"`;
341
+ register the returned bucket and uploaded object key in `Metadata.storages`. To read a
342
+ TD file, use `DataClient.open_td(reference, provider="tos", bucket=bucket)`.
343
+ `endpoint=` changes the network endpoint without changing the signing region;
344
+ it cannot be combined with `use_internal_endpoint=True`.
345
+
346
+ ## Pre-annotation
347
+
348
+ `current_data_py.labeling.LabelingClient` wraps the pre-annotation RPCs
349
+ `CreateTask`, `ListTasks`, `ListItems`, `WriteResult`, and `UpdatePreAnnotation` with
350
+ ManagementClient's TLS, package default address, caller-owned channel, per-call
351
+ `timeout` (30 seconds by default, `None` disables it), and `metadata` conventions.
352
+ Errors propagate without retries. Protobuf types are available through
353
+ `service_pb2`, `task_pb2`, and `DataSelection` in the same module.
354
+
355
+ ```python
356
+ from current_data_py.labeling import LabelingClient, service_pb2, task_pb2
357
+
358
+ with LabelingClient() as labeling:
359
+ task = labeling.create_task(
360
+ "template-id", name="Pre-annotation",
361
+ input_sources=[task_pb2.InputSource(td_file=task_pb2.TdFileSource(data_id="data-id"))],
362
+ start_pre_annotation=True,
363
+ )
364
+ for item in labeling.list_items(task.task_id):
365
+ # Match question keys and option keys to the existing built-in template.
366
+ labeling.write_result(item.item_id, task_pb2.Result(question_results=[
367
+ task_pb2.QuestionResult(
368
+ question_key="quality",
369
+ choice=task_pb2.ChoiceResult(selected_option_keys=["ok"]),
370
+ ),
371
+ ]))
372
+ # On inference failure instead: write_result(item.item_id, pre_annotation_error="reason")
373
+ labeling.update_pre_annotation(
374
+ task.task_id, service_pb2.UpdatePreAnnotationRequest.ACTION_OPEN_HUMAN_REVIEW,
375
+ )
376
+ ```
377
+
378
+ Alternatively, pass `data_selection=DataSelection(...)` when creating a task.
379
+ Use `ACTION_ACCEPT_ALL_AND_COMPLETE` to accept all pre-annotations as final labels.
380
+ Use an existing built-in template; custom templates do not support pre-annotation.
381
+ The server validates inputs and task transitions.
382
+
383
+ `list_tasks` and `list_items` return repeated protobuf collections.
384
+ `create_task` and `update_pre_annotation` return full protobuf responses
385
+ (including item count and task ID, or status). `write_result` returns `None`.
386
+
339
387
  ## 打包
340
388
 
341
389
  ```bash
@@ -355,8 +403,11 @@ python -m pip install --force-reinstall dist/*.whl
355
403
  python -I tests/check_package.py
356
404
  python -I tests/check_environment.py
357
405
  python -I tests/check_management.py
406
+ python -I tests/check_labeling.py
358
407
  python -I tests/check_remote.py
359
408
  python -I tests/check_oss.py
409
+ # With the tos extra installed:
410
+ python -I tests/check_tos.py
360
411
  ```
361
412
 
362
413
  ## 推送
@@ -1,3 +1,17 @@
1
+ Metadata-Version: 2.4
2
+ Name: current-data-py
3
+ Version: 0.1.12
4
+ Summary: Current Data protobuf messages and schema metadata
5
+ Requires-Python: >=3.9
6
+ Description-Content-Type: text/markdown
7
+ Requires-Dist: protobuf>=3.20
8
+ Requires-Dist: turbodata==0.1.0
9
+ Requires-Dist: grpcio>=1.51.1
10
+ Requires-Dist: alibabacloud-oss-v2<2,>=1.4.0
11
+ Requires-Dist: tos<3,>=2.9
12
+ Requires-Dist: av>=12
13
+ Requires-Dist: pillow>=9
14
+
1
15
  # current-data-py
2
16
 
3
17
  ## Environment
@@ -156,9 +170,8 @@ retry or fall back to earlier RPC versions. Old servers return `UNIMPLEMENTED`;
156
170
  unconfigured destinations return `FAILED_PRECONDITION`. A response whose
157
171
  provider differs from the request raises `ValueError`.
158
172
 
159
- These methods only fetch credentials. Pass TOS credentials to a native TOS
160
- client and refresh them before expiration. `OSSClient`, `DataClient`, metadata
161
- registration and signed URLs have not been extended to TOS by this change.
173
+ These methods only fetch credentials. For file transfers and TD reads with automatic
174
+ credential refresh, see [TOS](#tos). Signed download URLs remain OSS-only.
162
175
 
163
176
  ## Remote TD reads
164
177
 
@@ -166,12 +179,11 @@ registration and signed URLs have not been extended to TOS by this change.
166
179
 
167
180
  > [!IMPORTANT]
168
181
  > **Video decoding requires PyAV, Pillow and the FFmpeg native libraries used by
169
- > PyAV.** `open_td()` enables decoding by default, but a base `current-data-py`
170
- > installation does not install the video dependencies. Install the `[decode]`
171
- > extra in every environment that decodes video, including DataLoader workers.
182
+ > PyAV.** `current-data-py` installs PyAV and Pillow by default. Install the package
183
+ > in every environment that decodes video, including DataLoader workers.
172
184
 
173
185
  ```bash
174
- python -m pip install --only-binary=av,pillow 'current-data-py[decode]'
186
+ python -m pip install --only-binary=av,pillow current-data-py
175
187
  ```
176
188
 
177
189
  - **Prebuilt wheels:** PyAV wheels bundle FFmpeg libraries on supported platforms;
@@ -258,8 +270,9 @@ to each call.
258
270
 
259
271
  ### Storage and credentials
260
272
 
261
- `DataClient.open_td()` accepts an optional `bucket` argument. Without it, reads
262
- use the management service's `OSS_READ_BUCKET` and `OSS_READ_PREFIX` defaults.
273
+ `DataClient.open_td()` accepts optional `bucket` and `provider` arguments. OSS
274
+ references without either selection retain the service's `OSS_READ_BUCKET` and
275
+ `OSS_READ_PREFIX` defaults. See [TOS](#tos) for provider selection.
263
276
  With it, the SDK uses `GetReadCredentialsV2` for whole-bucket credentials in the
264
277
  bucket's region, limited by the server's RAM role, and selects the first matching
265
278
  storage in the reference. An explicit bucket does not inherit the default prefix.
@@ -322,6 +335,69 @@ with ManagementClient(address=os.environ["MANAGEMENT_ADDRESS"]) as management:
322
335
  Deploy the V2 credential APIs and configure bucket regions before using reads
323
336
  or the upload `data_kind` option.
324
337
 
338
+ ## TOS
339
+
340
+ `TOSClient` obtains and refreshes V3 credentials from Management. Keep
341
+ `ManagementClient` open during transfers.
342
+
343
+ ```python
344
+ import os
345
+ from current_data_py.management import ManagementClient
346
+ from current_data_py.tos import TOSClient
347
+
348
+ with ManagementClient() as management:
349
+ with TOSClient(management, permission="read", bucket=os.environ["TOS_READ_BUCKET"],
350
+ endpoint="https://tos-accelerate.volces.com") as source:
351
+ source.download_file(os.environ["SOURCE_OBJECT_KEY"], "/tmp/input.h5")
352
+ ```
353
+
354
+ For uploads, use `permission="upload", data_type="YAM", data_kind="CONVERTED"`;
355
+ register the returned bucket and uploaded object key in `Metadata.storages`. To read a
356
+ TD file, use `DataClient.open_td(reference, provider="tos", bucket=bucket)`.
357
+ `endpoint=` changes the network endpoint without changing the signing region;
358
+ it cannot be combined with `use_internal_endpoint=True`.
359
+
360
+ ## Pre-annotation
361
+
362
+ `current_data_py.labeling.LabelingClient` wraps the pre-annotation RPCs
363
+ `CreateTask`, `ListTasks`, `ListItems`, `WriteResult`, and `UpdatePreAnnotation` with
364
+ ManagementClient's TLS, package default address, caller-owned channel, per-call
365
+ `timeout` (30 seconds by default, `None` disables it), and `metadata` conventions.
366
+ Errors propagate without retries. Protobuf types are available through
367
+ `service_pb2`, `task_pb2`, and `DataSelection` in the same module.
368
+
369
+ ```python
370
+ from current_data_py.labeling import LabelingClient, service_pb2, task_pb2
371
+
372
+ with LabelingClient() as labeling:
373
+ task = labeling.create_task(
374
+ "template-id", name="Pre-annotation",
375
+ input_sources=[task_pb2.InputSource(td_file=task_pb2.TdFileSource(data_id="data-id"))],
376
+ start_pre_annotation=True,
377
+ )
378
+ for item in labeling.list_items(task.task_id):
379
+ # Match question keys and option keys to the existing built-in template.
380
+ labeling.write_result(item.item_id, task_pb2.Result(question_results=[
381
+ task_pb2.QuestionResult(
382
+ question_key="quality",
383
+ choice=task_pb2.ChoiceResult(selected_option_keys=["ok"]),
384
+ ),
385
+ ]))
386
+ # On inference failure instead: write_result(item.item_id, pre_annotation_error="reason")
387
+ labeling.update_pre_annotation(
388
+ task.task_id, service_pb2.UpdatePreAnnotationRequest.ACTION_OPEN_HUMAN_REVIEW,
389
+ )
390
+ ```
391
+
392
+ Alternatively, pass `data_selection=DataSelection(...)` when creating a task.
393
+ Use `ACTION_ACCEPT_ALL_AND_COMPLETE` to accept all pre-annotations as final labels.
394
+ Use an existing built-in template; custom templates do not support pre-annotation.
395
+ The server validates inputs and task transitions.
396
+
397
+ `list_tasks` and `list_items` return repeated protobuf collections.
398
+ `create_task` and `update_pre_annotation` return full protobuf responses
399
+ (including item count and task ID, or status). `write_result` returns `None`.
400
+
325
401
  ## 打包
326
402
 
327
403
  ```bash
@@ -341,8 +417,11 @@ python -m pip install --force-reinstall dist/*.whl
341
417
  python -I tests/check_package.py
342
418
  python -I tests/check_environment.py
343
419
  python -I tests/check_management.py
420
+ python -I tests/check_labeling.py
344
421
  python -I tests/check_remote.py
345
422
  python -I tests/check_oss.py
423
+ # With the tos extra installed:
424
+ python -I tests/check_tos.py
346
425
  ```
347
426
 
348
427
  ## 推送
@@ -2,14 +2,18 @@ MANIFEST.in
2
2
  README.md
3
3
  __init__.py
4
4
  decoding.py
5
+ labeling.py
5
6
  management.py
6
7
  oss.py
7
8
  pyproject.toml
8
9
  setup.py
10
+ tos.py
9
11
  ./__init__.py
10
12
  ./decoding.py
13
+ ./labeling.py
11
14
  ./management.py
12
15
  ./oss.py
16
+ ./tos.py
13
17
  ./examples/__init__.py
14
18
  ./examples/list_schemas.py
15
19
  ./examples/read_remote_td.py
@@ -18,6 +22,7 @@ setup.py
18
22
  ./remote/__init__.py
19
23
  ./remote/client.py
20
24
  ./remote/source.py
25
+ ./remote/tos_source.py
21
26
  ./schema/__init__.py
22
27
  ./schema/message_metadata.py
23
28
  current_data_py.egg-info/PKG-INFO
@@ -33,9 +38,11 @@ examples/write_td.py
33
38
  remote/__init__.py
34
39
  remote/client.py
35
40
  remote/source.py
41
+ remote/tos_source.py
36
42
  schema/__init__.py
37
43
  schema/message_metadata.py
38
44
  tests/check_environment.py
45
+ tests/check_labeling.py
39
46
  tests/check_management.py
40
47
  tests/check_oss.py
41
48
  tests/check_package.py
@@ -2,7 +2,6 @@ protobuf>=3.20
2
2
  turbodata==0.1.0
3
3
  grpcio>=1.51.1
4
4
  alibabacloud-oss-v2<2,>=1.4.0
5
-
6
- [decode]
5
+ tos<3,>=2.9
7
6
  av>=12
8
7
  pillow>=9
@@ -67,12 +67,9 @@ class DecodedMessage:
67
67
  class _VideoStream:
68
68
  """Hold one topic's H.264 decoder for the lifetime of a forward iterator."""
69
69
  def __init__(self, *, keep_timestamp=None):
70
- """Load optional video dependencies and create a fresh decoder."""
71
- try:
72
- import av
73
- from PIL import Image # noqa: F401
74
- except ImportError as exc:
75
- raise ImportError("Video decoding requires pip install 'current-data-py[decode]'") from exc
70
+ """Load video dependencies and create a fresh decoder."""
71
+ import av
72
+ from PIL import Image # noqa: F401
76
73
  self.av = av
77
74
  self.codec = av.CodecContext.create("h264", "r")
78
75
  self.codec.thread_count = 1
@@ -0,0 +1,102 @@
1
+ """Synchronous client for LabelingService pre-annotation RPCs."""
2
+
3
+ from typing import Optional, Sequence, Tuple, Union
4
+
5
+ import grpc
6
+
7
+ import current_data_py as _package
8
+ from . import DEFAULT_ADDRESS
9
+ from ._proto.labeling.service import service_pb2, service_pb2_grpc
10
+ from ._proto.labeling.task import task_pb2
11
+ from ._proto.management.selection.selection_pb2 import DataSelection
12
+
13
+ _CallMetadata = Optional[Sequence[Tuple[str, Union[str, bytes]]]]
14
+
15
+ __all__ = [
16
+ "DEFAULT_ADDRESS", "LabelingClient", "DataSelection",
17
+ "service_pb2", "task_pb2",
18
+ ]
19
+
20
+
21
+ class LabelingClient:
22
+ """Use TLS and current_data_py.DEFAULT_ADDRESS, or a caller-owned channel.
23
+
24
+ RPC errors propagate unchanged without retries. Call metadata carries headers
25
+ such as authorization; timeouts are in seconds. Server-side validation governs
26
+ pre-annotation results and task transitions.
27
+ """
28
+
29
+ def __init__(self, channel: Optional[grpc.Channel] = None, *, address: Optional[str] = None):
30
+ self._owns_channel = channel is None
31
+ self._channel = (
32
+ grpc.secure_channel(
33
+ _package.DEFAULT_ADDRESS if address is None else address, grpc.ssl_channel_credentials(),
34
+ )
35
+ if channel is None else channel
36
+ )
37
+ self.stub = service_pb2_grpc.LabelingServiceStub(self._channel)
38
+
39
+ def close(self) -> None:
40
+ """Close the channel only when this client created it."""
41
+ if self._owns_channel:
42
+ self._channel.close()
43
+
44
+ def __enter__(self) -> "LabelingClient":
45
+ return self
46
+
47
+ def __exit__(self, *args) -> None:
48
+ self.close()
49
+
50
+ def create_task(
51
+ self, template_id: str, *, input_sources: Optional[Sequence[task_pb2.InputSource]] = None,
52
+ data_selection: Optional[DataSelection] = None, name: str = "",
53
+ start_pre_annotation: bool = False,
54
+ timeout: Optional[float] = 30, metadata: _CallMetadata = None,
55
+ ) -> service_pb2.CreateTaskResponse:
56
+ return self.stub.CreateTask(
57
+ service_pb2.CreateTaskRequest(
58
+ template_id=template_id, input_sources=input_sources,
59
+ data_selection=data_selection, name=name, start_pre_annotation=start_pre_annotation,
60
+ ),
61
+ timeout=timeout, metadata=metadata,
62
+ )
63
+
64
+ def list_tasks(
65
+ self, *,
66
+ timeout: Optional[float] = 30, metadata: _CallMetadata = None,
67
+ ) -> Sequence[task_pb2.TaskSummary]:
68
+ return self.stub.ListTasks(
69
+ service_pb2.ListTasksRequest(),
70
+ timeout=timeout, metadata=metadata,
71
+ ).tasks
72
+
73
+ def list_items(
74
+ self, task_id: str, *,
75
+ timeout: Optional[float] = 30, metadata: _CallMetadata = None,
76
+ ) -> Sequence[task_pb2.Item]:
77
+ return self.stub.ListItems(
78
+ service_pb2.ListItemsRequest(task_id=task_id),
79
+ timeout=timeout, metadata=metadata,
80
+ ).items
81
+
82
+ def write_result(
83
+ self, item_id: str, result: Optional[task_pb2.Result] = None, *,
84
+ pre_annotation_error: str = "",
85
+ timeout: Optional[float] = 30, metadata: _CallMetadata = None,
86
+ ) -> None:
87
+ self.stub.WriteResult(
88
+ service_pb2.WriteResultRequest(
89
+ item_id=item_id, result=result,
90
+ pre_annotation_error=pre_annotation_error,
91
+ ),
92
+ timeout=timeout, metadata=metadata,
93
+ )
94
+
95
+ def update_pre_annotation(
96
+ self, task_id: str, action: int, *,
97
+ timeout: Optional[float] = 30, metadata: _CallMetadata = None,
98
+ ) -> service_pb2.UpdatePreAnnotationResponse:
99
+ return self.stub.UpdatePreAnnotation(
100
+ service_pb2.UpdatePreAnnotationRequest(task_id=task_id, action=action),
101
+ timeout=timeout, metadata=metadata,
102
+ )
@@ -8,14 +8,13 @@ import current_data_py as _package
8
8
  from . import DEFAULT_ADDRESS # Compatibility export; configure current_data_py.DEFAULT_ADDRESS.
9
9
 
10
10
  from ._proto.management.metadata import metadata_pb2
11
- from ._proto.management.metadata.metadata_pb2 import Metadata, MetadataFilter
11
+ from ._proto.management.metadata.metadata_pb2 import Metadata, MetadataFilter, StorageProvider
12
12
  from ._proto.management.service import service_pb2, service_pb2_grpc
13
13
  from ._proto.management.service.service_pb2 import (
14
14
  DataKind,
15
15
  GetReadCredentialsResponse,
16
16
  GetUploadCredentialsResponse,
17
17
  StorageCredentials,
18
- StorageProvider,
19
18
  )
20
19
 
21
20
 
@@ -30,9 +29,10 @@ __all__ = [
30
29
 
31
30
  def _storage_provider(value: Union[int, str]) -> int:
32
31
  if isinstance(value, str):
32
+ value = value.upper()
33
33
  name = value if value.startswith("STORAGE_PROVIDER_") else "STORAGE_PROVIDER_" + value
34
34
  value = StorageProvider.Value(name)
35
- if isinstance(value, bool) or value not in (service_pb2.STORAGE_PROVIDER_OSS, service_pb2.STORAGE_PROVIDER_TOS):
35
+ if isinstance(value, bool) or value not in (metadata_pb2.STORAGE_PROVIDER_OSS, metadata_pb2.STORAGE_PROVIDER_TOS):
36
36
  raise ValueError("provider must explicitly select OSS or TOS")
37
37
  return value
38
38