current-data-py 0.1.2__tar.gz → 0.1.5__tar.gz

current_data_py-0.1.5/MANIFEST.in

@@ -0,0 +1,4 @@
+include tests/check_package.py
+include tests/check_remote.py
+include tests/check_management.py
+include tests/check_oss.py

current_data_py-0.1.5/PKG-INFO

@@ -0,0 +1,98 @@
+Metadata-Version: 2.4
+Name: current-data-py
+Version: 0.1.5
+Summary: Current Data protobuf messages and schema metadata
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+Requires-Dist: protobuf>=3.20
+Requires-Dist: turbodata==0.1.0
+Requires-Dist: grpcio>=1.51.1
+Requires-Dist: alibabacloud-oss-v2<2,>=1.4.0
+
+# current-data-py
+
+## Management
+
+`ManagementClient()` connects to `data.current-robotics.work:443` over TLS.
+Pass `address="host:443"` to override it, or supply your own gRPC channel.
+
+```python
+from current_data_py.management import ManagementClient, Metadata, MetadataFilter
+
+with ManagementClient() as client:
+    upload = client.get_upload_credentials(data_type="CURRENT_V2")
+    read = client.get_read_credentials(bucket="data-infra-raw-data")
+
+    # Register an already uploaded file; replace the example file details.
+    record = client.register_data(Metadata(
+        name="recording",
+        data_type="DATA_TYPE_CURRENT_V2",
+        storages=[{"bucket": "data-infra-202607", "object_key": "raw_converted/recording.td"}],
+        file_info={"size_bytes": 1234, "checksum": "sha256:<file-hash>"},
+    ))
+    for item in client.stream_data_references(MetadataFilter(name_prefix="recording")):
+        print(item.id, item.storages)
+```
+
+`Metadata`, `MetadataFilter`, and responses are Protobuf objects exported by the
+SDK. Registration only saves metadata; it does not upload files. Credential
+calls return fresh credentials without caching. Calls accept `timeout=` and
+`metadata=` for gRPC headers.
+
+## OSS
+
+```python
+from current_data_py.management import ManagementClient
+from current_data_py.oss import OSSClient
+
+with ManagementClient() as management:
+    with OSSClient(management, permission="read", bucket="data-infra-raw-data") as source:
+        source.download_file("collections/recording/input.h5", "/tmp/input.h5")
+
+    # Convert the input into /tmp/output.td before uploading.
+    with OSSClient(management, permission="upload", data_type="CURRENT_V2") as destination:
+        key = destination.object_prefix + "recording/output.td"
+        destination.upload_file(key, "/tmp/output.td")
+        print(destination.bucket, key)  # Use these values when registering metadata.
+```
+
+- Read access covers the requested bucket, subject to server permissions.
+  Uploads default to `data_type="RAW"`; the server chooses the bucket and prefix.
+- Credentials refresh automatically. Expired tokens trigger one refresh and retry.
+- Uploads reject overwrites by default; set `forbid_overwrite=False` to replace.
+  Downloads replace the local file only on success; its parent directory must exist.
+- Uploads use a single PutObject request ([maximum 5 GiB](https://www.alibabacloud.com/help/en/oss/developer-reference/upload-files-using-oss-sdk-for-python-v2/)); multipart upload is not supported.
+- Supply `region=` for custom upload endpoints. Deploy the current management
+  credential API before using the new bucket and data-type arguments.
+
+## 打包
+
+```bash
+cd current_data_py  # 从仓库根目录执行
+# 发布新版本前，更新 pyproject.toml 中的 version
+python -m pip install build twine
+rm -rf build dist
+python -m build
+```
+
+## 测试
+
+```bash
+python -m twine check dist/*
+python tests/check_distribution.py
+python -m pip install --force-reinstall dist/*.whl
+python -I tests/check_package.py
+python -I tests/check_management.py
+python -I tests/check_remote.py
+python -I tests/check_oss.py
+```
+
+## 推送
+
+```bash
+# TestPyPI
+python -m twine upload --repository testpypi dist/*
+
+# PyPI
+python -m twine upload dist/*
+```

current_data_py-0.1.5/README.md

@@ -0,0 +1,87 @@
+# current-data-py
+
+## Management
+
+`ManagementClient()` connects to `data.current-robotics.work:443` over TLS.
+Pass `address="host:443"` to override it, or supply your own gRPC channel.
+
+```python
+from current_data_py.management import ManagementClient, Metadata, MetadataFilter
+
+with ManagementClient() as client:
+    upload = client.get_upload_credentials(data_type="CURRENT_V2")
+    read = client.get_read_credentials(bucket="data-infra-raw-data")
+
+    # Register an already uploaded file; replace the example file details.
+    record = client.register_data(Metadata(
+        name="recording",
+        data_type="DATA_TYPE_CURRENT_V2",
+        storages=[{"bucket": "data-infra-202607", "object_key": "raw_converted/recording.td"}],
+        file_info={"size_bytes": 1234, "checksum": "sha256:<file-hash>"},
+    ))
+    for item in client.stream_data_references(MetadataFilter(name_prefix="recording")):
+        print(item.id, item.storages)
+```
+
+`Metadata`, `MetadataFilter`, and responses are Protobuf objects exported by the
+SDK. Registration only saves metadata; it does not upload files. Credential
+calls return fresh credentials without caching. Calls accept `timeout=` and
+`metadata=` for gRPC headers.
+
+## OSS
+
+```python
+from current_data_py.management import ManagementClient
+from current_data_py.oss import OSSClient
+
+with ManagementClient() as management:
+    with OSSClient(management, permission="read", bucket="data-infra-raw-data") as source:
+        source.download_file("collections/recording/input.h5", "/tmp/input.h5")
+
+    # Convert the input into /tmp/output.td before uploading.
+    with OSSClient(management, permission="upload", data_type="CURRENT_V2") as destination:
+        key = destination.object_prefix + "recording/output.td"
+        destination.upload_file(key, "/tmp/output.td")
+        print(destination.bucket, key)  # Use these values when registering metadata.
+```
+
+- Read access covers the requested bucket, subject to server permissions.
+  Uploads default to `data_type="RAW"`; the server chooses the bucket and prefix.
+- Credentials refresh automatically. Expired tokens trigger one refresh and retry.
+- Uploads reject overwrites by default; set `forbid_overwrite=False` to replace.
+  Downloads replace the local file only on success; its parent directory must exist.
+- Uploads use a single PutObject request ([maximum 5 GiB](https://www.alibabacloud.com/help/en/oss/developer-reference/upload-files-using-oss-sdk-for-python-v2/)); multipart upload is not supported.
+- Supply `region=` for custom upload endpoints. Deploy the current management
+  credential API before using the new bucket and data-type arguments.
+
+## 打包
+
+```bash
+cd current_data_py  # 从仓库根目录执行
+# 发布新版本前，更新 pyproject.toml 中的 version
+python -m pip install build twine
+rm -rf build dist
+python -m build
+```
+
+## 测试
+
+```bash
+python -m twine check dist/*
+python tests/check_distribution.py
+python -m pip install --force-reinstall dist/*.whl
+python -I tests/check_package.py
+python -I tests/check_management.py
+python -I tests/check_remote.py
+python -I tests/check_oss.py
+```
+
+## 推送
+
+```bash
+# TestPyPI
+python -m twine upload --repository testpypi dist/*
+
+# PyPI
+python -m twine upload dist/*
+```

{current_data_py-0.1.2 → current_data_py-0.1.5}/__init__.py

@@ -1,6 +1,8 @@
-"""Protobuf message schemas and metadata for Current Data."""
-
-from .schema.message_metadata import descriptor_set, topic_metadata
-from .schema import SCHEMA_TO_CLASS
-
-__all__ = ["SCHEMA_TO_CLASS", "descriptor_set", "topic_metadata"]
+"""Protobuf message schemas and metadata for Current Data."""
+
+from .schema.message_metadata import descriptor_set, topic_metadata
+from .schema import SCHEMA_TO_CLASS
+
+DEFAULT_ADDRESS = "data.current-robotics.work:443"
+
+__all__ = ["DEFAULT_ADDRESS", "SCHEMA_TO_CLASS", "descriptor_set", "topic_metadata"]

current_data_py-0.1.5/current_data_py.egg-info/PKG-INFO

@@ -0,0 +1,98 @@
+Metadata-Version: 2.4
+Name: current-data-py
+Version: 0.1.5
+Summary: Current Data protobuf messages and schema metadata
+Requires-Python: >=3.9
+Description-Content-Type: text/markdown
+Requires-Dist: protobuf>=3.20
+Requires-Dist: turbodata==0.1.0
+Requires-Dist: grpcio>=1.51.1
+Requires-Dist: alibabacloud-oss-v2<2,>=1.4.0
+
+# current-data-py
+
+## Management
+
+`ManagementClient()` connects to `data.current-robotics.work:443` over TLS.
+Pass `address="host:443"` to override it, or supply your own gRPC channel.
+
+```python
+from current_data_py.management import ManagementClient, Metadata, MetadataFilter
+
+with ManagementClient() as client:
+    upload = client.get_upload_credentials(data_type="CURRENT_V2")
+    read = client.get_read_credentials(bucket="data-infra-raw-data")
+
+    # Register an already uploaded file; replace the example file details.
+    record = client.register_data(Metadata(
+        name="recording",
+        data_type="DATA_TYPE_CURRENT_V2",
+        storages=[{"bucket": "data-infra-202607", "object_key": "raw_converted/recording.td"}],
+        file_info={"size_bytes": 1234, "checksum": "sha256:<file-hash>"},
+    ))
+    for item in client.stream_data_references(MetadataFilter(name_prefix="recording")):
+        print(item.id, item.storages)
+```
+
+`Metadata`, `MetadataFilter`, and responses are Protobuf objects exported by the
+SDK. Registration only saves metadata; it does not upload files. Credential
+calls return fresh credentials without caching. Calls accept `timeout=` and
+`metadata=` for gRPC headers.
+
+## OSS
+
+```python
+from current_data_py.management import ManagementClient
+from current_data_py.oss import OSSClient
+
+with ManagementClient() as management:
+    with OSSClient(management, permission="read", bucket="data-infra-raw-data") as source:
+        source.download_file("collections/recording/input.h5", "/tmp/input.h5")
+
+    # Convert the input into /tmp/output.td before uploading.
+    with OSSClient(management, permission="upload", data_type="CURRENT_V2") as destination:
+        key = destination.object_prefix + "recording/output.td"
+        destination.upload_file(key, "/tmp/output.td")
+        print(destination.bucket, key)  # Use these values when registering metadata.
+```
+
+- Read access covers the requested bucket, subject to server permissions.
+  Uploads default to `data_type="RAW"`; the server chooses the bucket and prefix.
+- Credentials refresh automatically. Expired tokens trigger one refresh and retry.
+- Uploads reject overwrites by default; set `forbid_overwrite=False` to replace.
+  Downloads replace the local file only on success; its parent directory must exist.
+- Uploads use a single PutObject request ([maximum 5 GiB](https://www.alibabacloud.com/help/en/oss/developer-reference/upload-files-using-oss-sdk-for-python-v2/)); multipart upload is not supported.
+- Supply `region=` for custom upload endpoints. Deploy the current management
+  credential API before using the new bucket and data-type arguments.
+
+## 打包
+
+```bash
+cd current_data_py  # 从仓库根目录执行
+# 发布新版本前，更新 pyproject.toml 中的 version
+python -m pip install build twine
+rm -rf build dist
+python -m build
+```
+
+## 测试
+
+```bash
+python -m twine check dist/*
+python tests/check_distribution.py
+python -m pip install --force-reinstall dist/*.whl
+python -I tests/check_package.py
+python -I tests/check_management.py
+python -I tests/check_remote.py
+python -I tests/check_oss.py
+```
+
+## 推送
+
+```bash
+# TestPyPI
+python -m twine upload --repository testpypi dist/*
+
+# PyPI
+python -m twine upload dist/*
+```

{current_data_py-0.1.2 → current_data_py-0.1.5}/current_data_py.egg-info/SOURCES.txt

@@ -1,9 +1,13 @@
 MANIFEST.in
 README.md
 __init__.py
+management.py
+oss.py
 pyproject.toml
 setup.py
 ./__init__.py
+./management.py
+./oss.py
 ./examples/__init__.py
 ./examples/list_schemas.py
 ./examples/read_remote_td.py
@@ -29,5 +33,7 @@ remote/client.py
 remote/source.py
 schema/__init__.py
 schema/message_metadata.py
+tests/check_management.py
+tests/check_oss.py
 tests/check_package.py
 tests/check_remote.py

{current_data_py-0.1.2 → current_data_py-0.1.5}/current_data_py.egg-info/requires.txt

@@ -1,6 +1,4 @@
 protobuf>=3.20
 turbodata==0.1.0
-
-[remote]
 grpcio>=1.51.1
 alibabacloud-oss-v2<2,>=1.4.0

{current_data_py-0.1.2 → current_data_py-0.1.5}/examples/__init__.py

@@ -1 +1 @@
-"""Runnable examples for Current Data."""
+"""Runnable examples for Current Data."""

{current_data_py-0.1.2 → current_data_py-0.1.5}/examples/list_schemas.py

@@ -1,7 +1,7 @@
-from ..schema import SCHEMA_TO_CLASS
-
-
-if __name__ == "__main__":
-    print("SCHEMA_TO_CLASS:")
-    for schema, msg_class in SCHEMA_TO_CLASS.items():
-        print(f"  {schema}: {msg_class}")
+from ..schema import SCHEMA_TO_CLASS
+
+
+if __name__ == "__main__":
+    print("SCHEMA_TO_CLASS:")
+    for schema, msg_class in SCHEMA_TO_CLASS.items():
+        print(f"  {schema}: {msg_class}")

{current_data_py-0.1.2 → current_data_py-0.1.5}/examples/read_remote_td.py

@@ -1,48 +1,54 @@
-"""Read up to ten messages from the first matching remote TD file."""
-
-import argparse
-from contextlib import ExitStack, closing
-from itertools import islice
-
-from google.protobuf.text_format import MessageToString
-
-from ..remote import DataClient, MetadataFilter
-from ..schema import SCHEMA_TO_CLASS
-
-
-def main() -> None:
-    parser = argparse.ArgumentParser(description=__doc__)
-    parser.add_argument("--name-prefix", default="", help="Filter recordings by name prefix")
-    parser.add_argument("--topic", default="/head/pose/command")
-    parser.add_argument("--start-timestamp", type=int, default=0, help="Inclusive start timestamp (ns)")
-    parser.add_argument("--end-timestamp", type=int, default=2**63 - 1, help="Inclusive end timestamp (ns)")
-    args = parser.parse_args()
-    if not 0 <= args.start_timestamp <= args.end_timestamp <= 2**63 - 1:
-        parser.error("timestamps must satisfy 0 <= start <= end <= 2**63 - 1")
-
-    with ExitStack() as stack:
-        client = stack.enter_context(DataClient())
-        with closing(client.iter_data(MetadataFilter(name_prefix=args.name_prefix), timeout=30)) as items:
-            metadata = next(items, None)
-        if metadata is None:
-            print("No matching data")
-            return
-
-        with client.open_td(metadata) as reader:
-            schemas = {
-                topic.name: topic.metadata["schema_name"]
-                for group in reader.summary().topics_infos
-                for topic in group.topic_metadatas
-            }
-            messages = reader.read_messages(
-                topic_names=[args.topic],
-                start_timestamp=args.start_timestamp,
-                end_timestamp=args.end_timestamp,
-            )
-            for message in islice(messages, 10):
-                decoded = SCHEMA_TO_CLASS[schemas[message.topic_name]].FromString(message.data)
-                print(metadata.data_id, message.timestamp, message.topic_name, MessageToString(decoded, as_one_line=True))
-
-
-if __name__ == "__main__":
-    main()
+"""Read up to ten messages from the first matching remote TD file."""
+
+import argparse
+from contextlib import ExitStack, closing
+from itertools import islice
+
+import grpc
+from google.protobuf.text_format import MessageToString
+
+from ..remote import DataClient, MetadataFilter
+from ..schema import SCHEMA_TO_CLASS
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("address", nargs="?", default=None, help="gRPC service address (host:port)")
+    parser.add_argument("--insecure", action="store_true", help="Use a plaintext gRPC connection instead of TLS")
+    parser.add_argument("--name-prefix", default="", help="Filter recordings by name prefix")
+    parser.add_argument("--topic", default="/head/pose/command")
+    parser.add_argument("--start-timestamp", type=int, default=0, help="Inclusive start timestamp (ns)")
+    parser.add_argument("--end-timestamp", type=int, default=2**63 - 1, help="Inclusive end timestamp (ns)")
+    args = parser.parse_args()
+    if not 0 <= args.start_timestamp <= args.end_timestamp <= 2**63 - 1:
+        parser.error("timestamps must satisfy 0 <= start <= end <= 2**63 - 1")
+
+    with ExitStack() as stack:
+        channel = grpc.insecure_channel(args.address) if args.insecure else None
+        if channel is not None:
+            stack.callback(channel.close)
+        client = stack.enter_context(DataClient(channel, address=args.address))
+        with closing(client.iter_data(MetadataFilter(name_prefix=args.name_prefix), timeout=30)) as items:
+            metadata = next(items, None)
+        if metadata is None:
+            print("No matching data")
+            return
+
+        with client.open_td(metadata) as reader:
+            schemas = {
+                topic.name: topic.metadata["schema_name"]
+                for group in reader.summary().topics_infos
+                for topic in group.topic_metadatas
+            }
+            messages = reader.read_messages(
+                topic_names=[args.topic],
+                start_timestamp=args.start_timestamp,
+                end_timestamp=args.end_timestamp,
+            )
+            for message in islice(messages, 10):
+                decoded = SCHEMA_TO_CLASS[schemas[message.topic_name]].FromString(message.data)
+                print(metadata.id, message.timestamp, message.topic_name, MessageToString(decoded, as_one_line=True))
+
+
+if __name__ == "__main__":
+    main()

{current_data_py-0.1.2 → current_data_py-0.1.5}/examples/read_td.py

@@ -1,30 +1,30 @@
-import argparse
-from pathlib import Path
-
-from turbodata import FileReadSource, Reader
-
-from ..schema import SCHEMA_TO_CLASS
-
-
-def main() -> None:
-    parser = argparse.ArgumentParser(description="Read protobuf messages from a TD file")
-    parser.add_argument("path", type=Path)
-    args = parser.parse_args()
-
-    with FileReadSource(str(args.path)) as source:
-        reader = Reader(source)
-        schemas = {
-            topic.name: topic.metadata["schema_name"]
-            for group in reader.summary().topics_infos
-            for topic in group.topic_metadatas
-        }
-
-        for message in reader.read_messages(topic_names=["/head/pose/command"]):
-            schema = schemas[message.topic_name]
-            msg = SCHEMA_TO_CLASS[schema]()
-            msg.ParseFromString(message.data)
-            print(message.timestamp, message.topic_name, schema, msg.position.y)
-
-
-if __name__ == "__main__":
-    main()
+import argparse
+from pathlib import Path
+
+from turbodata import FileReadSource, Reader
+
+from ..schema import SCHEMA_TO_CLASS
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Read protobuf messages from a TD file")
+    parser.add_argument("path", type=Path)
+    args = parser.parse_args()
+
+    with FileReadSource(str(args.path)) as source:
+        reader = Reader(source)
+        schemas = {
+            topic.name: topic.metadata["schema_name"]
+            for group in reader.summary().topics_infos
+            for topic in group.topic_metadatas
+        }
+
+        for message in reader.read_messages(topic_names=["/head/pose/command"]):
+            schema = schemas[message.topic_name]
+            msg = SCHEMA_TO_CLASS[schema]()
+            msg.ParseFromString(message.data)
+            print(message.timestamp, message.topic_name, schema, msg.position.y)
+
+
+if __name__ == "__main__":
+    main()

{current_data_py-0.1.2 → current_data_py-0.1.5}/examples/write_td.py

@@ -1,30 +1,30 @@
-import argparse
-import time
-from pathlib import Path
-
-from turbodata import Writer
-
-from ..schema.message_metadata import topic_metadata
-from ..schema import Pose
-
-
-TOPIC = "/head/pose/command"
-
-
-def main() -> None:
-    parser = argparse.ArgumentParser(description="Write a protobuf message to a TD file")
-    parser.add_argument("path", type=Path)
-    args = parser.parse_args()
-
-    message = Pose()
-    message.position.y = 1.0
-
-    with args.path.open("wb") as output, Writer(output) as writer:
-        writer.open_topics([TOPIC], [topic_metadata(type(message))])
-        timestamp = time.time_ns()
-        writer.write_message(TOPIC, message.SerializeToString(), timestamp)
-        writer.close_topic()
-
-
-if __name__ == "__main__":
-    main()
+import argparse
+import time
+from pathlib import Path
+
+from turbodata import Writer
+
+from ..schema.message_metadata import topic_metadata
+from ..schema import Pose
+
+
+TOPIC = "/head/pose/command"
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Write a protobuf message to a TD file")
+    parser.add_argument("path", type=Path)
+    args = parser.parse_args()
+
+    message = Pose()
+    message.position.y = 1.0
+
+    with args.path.open("wb") as output, Writer(output) as writer:
+        writer.open_topics([TOPIC], [topic_metadata(type(message))])
+        timestamp = time.time_ns()
+        writer.write_message(TOPIC, message.SerializeToString(), timestamp)
+        writer.close_topic()
+
+
+if __name__ == "__main__":
+    main()