ptars 0.0.1rc1__tar.gz → 0.0.2rc2__tar.gz

{ptars-0.0.1rc1 → ptars-0.0.2rc2}/Cargo.lock

@@ -306,9 +306,9 @@ checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9"
 
 [[package]]
 name = "cc"
-version = "1.0.92"
+version = "1.0.95"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2678b2e3449475e95b0aa6f9b506a28e61b3dc8996592b983695e8ebb58a8b41"
+checksum = "d32a725bc159af97c3e629873bb9f88fb8cf8a4867175f76dc987815ea07c83b"
 
 [[package]]
 name = "cfg-if"
@@ -318,16 +318,16 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
 [[package]]
 name = "chrono"
-version = "0.4.37"
+version = "0.4.38"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8a0d04d43504c61aa6c7531f1871dd0d418d91130162063b789da00fd7057a5e"
+checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401"
 dependencies = [
  "android-tzdata",
  "iana-time-zone",
  "js-sys",
  "num-traits",
  "wasm-bindgen",
- "windows-targets 0.52.4",
+ "windows-targets 0.52.5",
 ]
 
 [[package]]
@@ -602,9 +602,9 @@ dependencies = [
 
 [[package]]
 name = "num"
-version = "0.4.1"
+version = "0.4.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b05180d69e3da0e530ba2a1dae5110317e49e3b7f3d41be227dc5f92e49ee7af"
+checksum = "3135b08af27d103b0a51f2ae0f8632117b7b185ccf931445affa8df530576a41"
 dependencies = [
  "num-bigint",
  "num-complex",
@@ -713,9 +713,9 @@ checksum = "7170ef9988bc169ba16dd36a7fa041e5c4cbeb6a35b76d4c03daded371eae7c0"
 
 [[package]]
 name = "proc-macro2"
-version = "1.0.79"
+version = "1.0.81"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e835ff2298f5721608eb1a980ecaee1aef2c132bf95ecc026a11b7bf3c01c02e"
+checksum = "3d1597b0c024618f09a9c3b8655b7e430397a36d23fdafec26d6965e9eec3eba"
 dependencies = [
  "unicode-ident",
 ]
@@ -742,7 +742,7 @@ dependencies = [
 
 [[package]]
 name = "ptarslib"
-version = "0.0.1-rc1"
+version = "0.0.2-rc2"
 dependencies = [
  "arrow",
  "arrow-array",
@@ -817,9 +817,9 @@ dependencies = [
 
 [[package]]
 name = "quote"
-version = "1.0.35"
+version = "1.0.36"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef"
+checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7"
 dependencies = [
  "proc-macro2",
 ]
@@ -891,18 +891,18 @@ checksum = "92d43fe69e652f3df9bdc2b85b2854a0825b86e4fb76bc44d945137d053639ca"
 
 [[package]]
 name = "serde"
-version = "1.0.197"
+version = "1.0.198"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3fb1c873e1b9b056a4dc4c0c198b24c3ffa059243875552b2bd0933b1aee4ce2"
+checksum = "9846a40c979031340571da2545a4e5b7c4163bdae79b301d5f86d03979451fcc"
 dependencies = [
  "serde_derive",
 ]
 
 [[package]]
 name = "serde_derive"
-version = "1.0.197"
+version = "1.0.198"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7eb0b34b42edc17f6b7cac84a52a1c5f0e1bb2227e997ca9011ea3dd34e8610b"
+checksum = "e88edab869b01783ba905e7d0153f9fc1a6505a96e4ad3018011eedb838566d9"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -911,9 +911,9 @@ dependencies = [
 
 [[package]]
 name = "serde_json"
-version = "1.0.115"
+version = "1.0.116"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "12dc5c46daa8e9fdf4f5e71b6cf9a53f2487da0e86e55808e2d35539666497dd"
+checksum = "3e17db7126d17feb94eb3fad46bf1a96b034e8aacbc2e775fe81505f8b0b2813"
 dependencies = [
  "itoa",
  "ryu",
@@ -934,9 +934,9 @@ checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
 
 [[package]]
 name = "syn"
-version = "2.0.58"
+version = "2.0.60"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "44cfb93f38070beee36b3fef7d4f5a16f27751d94b187b666a5cc5e9b0d30687"
+checksum = "909518bc7b1c9b779f1bbf07f2929d35af9f0f37e47c6e9ef7f9dddc1e1821f3"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -951,18 +951,18 @@ checksum = "e1fc403891a21bcfb7c37834ba66a547a8f402146eba7265b5a6d88059c9ff2f"
 
 [[package]]
 name = "thiserror"
-version = "1.0.58"
+version = "1.0.59"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "03468839009160513471e86a034bb2c5c0e4baae3b43f79ffc55c4a5427b3297"
+checksum = "f0126ad08bff79f29fc3ae6a55cc72352056dfff61e3ff8bb7129476d44b23aa"
 dependencies = [
  "thiserror-impl",
 ]
 
 [[package]]
 name = "thiserror-impl"
-version = "1.0.58"
+version = "1.0.59"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c61f3ba182994efc43764a46c018c347bc492c79f024e705f46567b418f6d4f7"
+checksum = "d1cd413b5d558b4c5bf3680e324a6fa5014e7b7c067a51e69dbdf47eb7148b66"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -1062,7 +1062,7 @@ version = "0.52.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9"
 dependencies = [
- "windows-targets 0.52.4",
+ "windows-targets 0.52.5",
 ]
 
 [[package]]
@@ -1082,17 +1082,18 @@ dependencies = [
 
 [[package]]
 name = "windows-targets"
-version = "0.52.4"
+version = "0.52.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7dd37b7e5ab9018759f893a1952c9420d060016fc19a472b4bb20d1bdd694d1b"
+checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb"
 dependencies = [
- "windows_aarch64_gnullvm 0.52.4",
- "windows_aarch64_msvc 0.52.4",
- "windows_i686_gnu 0.52.4",
- "windows_i686_msvc 0.52.4",
- "windows_x86_64_gnu 0.52.4",
- "windows_x86_64_gnullvm 0.52.4",
- "windows_x86_64_msvc 0.52.4",
+ "windows_aarch64_gnullvm 0.52.5",
+ "windows_aarch64_msvc 0.52.5",
+ "windows_i686_gnu 0.52.5",
+ "windows_i686_gnullvm",
+ "windows_i686_msvc 0.52.5",
+ "windows_x86_64_gnu 0.52.5",
+ "windows_x86_64_gnullvm 0.52.5",
+ "windows_x86_64_msvc 0.52.5",
 ]
 
 [[package]]
@@ -1103,9 +1104,9 @@ checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8"
 
 [[package]]
 name = "windows_aarch64_gnullvm"
-version = "0.52.4"
+version = "0.52.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bcf46cf4c365c6f2d1cc93ce535f2c8b244591df96ceee75d8e83deb70a9cac9"
+checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263"
 
 [[package]]
 name = "windows_aarch64_msvc"
@@ -1115,9 +1116,9 @@ checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc"
 
 [[package]]
 name = "windows_aarch64_msvc"
-version = "0.52.4"
+version = "0.52.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "da9f259dd3bcf6990b55bffd094c4f7235817ba4ceebde8e6d11cd0c5633b675"
+checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6"
 
 [[package]]
 name = "windows_i686_gnu"
@@ -1127,9 +1128,15 @@ checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e"
 
 [[package]]
 name = "windows_i686_gnu"
-version = "0.52.4"
+version = "0.52.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670"
+
+[[package]]
+name = "windows_i686_gnullvm"
+version = "0.52.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b474d8268f99e0995f25b9f095bc7434632601028cf86590aea5c8a5cb7801d3"
+checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9"
 
 [[package]]
 name = "windows_i686_msvc"
@@ -1139,9 +1146,9 @@ checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406"
 
 [[package]]
 name = "windows_i686_msvc"
-version = "0.52.4"
+version = "0.52.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1515e9a29e5bed743cb4415a9ecf5dfca648ce85ee42e15873c3cd8610ff8e02"
+checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf"
 
 [[package]]
 name = "windows_x86_64_gnu"
@@ -1151,9 +1158,9 @@ checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e"
 
 [[package]]
 name = "windows_x86_64_gnu"
-version = "0.52.4"
+version = "0.52.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5eee091590e89cc02ad514ffe3ead9eb6b660aedca2183455434b93546371a03"
+checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9"
 
 [[package]]
 name = "windows_x86_64_gnullvm"
@@ -1163,9 +1170,9 @@ checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc"
 
 [[package]]
 name = "windows_x86_64_gnullvm"
-version = "0.52.4"
+version = "0.52.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "77ca79f2451b49fa9e2af39f0747fe999fcda4f5e241b2898624dca97a1f2177"
+checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596"
 
 [[package]]
 name = "windows_x86_64_msvc"
@@ -1175,9 +1182,9 @@ checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538"
 
 [[package]]
 name = "windows_x86_64_msvc"
-version = "0.52.4"
+version = "0.52.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "32b752e52a2da0ddfbdbcc6fceadfeede4c939ed16d13e648833a61dfb611ed8"
+checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0"
 
 [[package]]
 name = "zerocopy"

{ptars-0.0.1rc1 → ptars-0.0.2rc2}/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "ptarslib"
-version = "0.0.1-rc1"
+version = "0.0.2-rc2"
 authors = ["0x26res <0x26res@gmail.net>"]
 edition = "2021"
 description = "Fast python conversion from protobuf to arrow using rust"
@@ -27,8 +27,8 @@ pyo3 = { version = "0.20.3" }
 arrow = { version = "51.0.0", features = ["pyarrow"] }
 arrow-array = { version = "51.0.0" }
 arrow-schema = { version = "51.0.0" }
-protobuf = { version = "3.3.0" }
-chrono = { version = "0.4.31" }
+protobuf = { version = "3.4.0" }
+chrono = { version = "0.4.38" }
 [features]
 extension-module = ["pyo3/extension-module"]
 default = ["extension-module"]

{ptars-0.0.1rc1 → ptars-0.0.2rc2}/DEVELOPMENT.md

@@ -20,10 +20,18 @@ export CARGO_BUILD_TARGET=x86_64-apple-darwin
 ## Testing
 
 ```shell
-cargo build && maturin develop && RUST_BACKTRACE=1  pytest test
+cargo build && maturin develop && RUST_BACKTRACE=1 pytest python/test
 ```
 
 
+## Releasing
+
+- Update the version in [Cargo.toml](./Cargo.toml)
+- Update cargo lock file `cargo generate-lockfile`
+- Update the version in [pyproject.toml](./pyproject.toml)
+- Update the poetry lock file `poetry lock --no-update`
+- Tag and push the tag (don't forget to preprend a `v` to the version when tagging)
+
 ## TODO
 
 - [ ] arrow to proto

{ptars-0.0.1rc1 → ptars-0.0.2rc2}/PKG-INFO

@@ -1,9 +1,10 @@
 Metadata-Version: 2.3
 Name: ptars
-Version: 0.0.1rc1
+Version: 0.0.2rc2
 Classifier: Programming Language :: Rust
 Classifier: Operating System :: POSIX :: Linux
 Requires-Dist: protobuf >3
+Requires-Dist: pyarrow >15
 License-File: LICENSE
 Summary: Fast python conversion from protobuf to arrow using rust
 Home-Page: https://github.com/0x26res/ptars

ptars-0.0.2rc2/protos/ptars_protos/imported.proto

@@ -0,0 +1,20 @@
+syntax = "proto3";
+
+import "google/protobuf/timestamp.proto";
+import "google/protobuf/wrappers.proto";
+import "google/type/date.proto";
+
+
+enum ImportedEnum {
+  UNKNOWN_CORE_ENUM = 0;
+  CORE_ENUM_FOO = 1;
+  CORE_ENUM_BAR = 2;
+}
+
+message ImportedMessage {
+  string string_value = 1;
+  ImportedEnum imported_enum = 2;
+  google.protobuf.DoubleValue double_value = 3;
+  google.type.Date date  = 4;
+  google.protobuf.Timestamp timestamp  = 5;
+}

ptars-0.0.2rc2/protos/ptars_protos/importer.proto

@@ -0,0 +1,16 @@
+syntax = "proto3";
+
+import "ptars_protos/imported.proto";
+import "google/protobuf/timestamp.proto";
+import "google/protobuf/wrappers.proto";
+import "google/type/date.proto";
+
+
+message ImporterMessage {
+  string string_value = 1;
+  ImportedEnum imported_enum = 2;
+  google.protobuf.DoubleValue double_value = 3;
+  google.type.Date date  = 4;
+  google.protobuf.Timestamp timestamp  = 5;
+  ImportedMessage imported_message = 6;
+}

{ptars-0.0.1rc1 → ptars-0.0.2rc2}/pyproject.toml

@@ -11,11 +11,12 @@ classifiers = [
 requires-python = ">=3.8"
 dependencies = [
     'protobuf > 3',
+    "pyarrow > 15",
 ]
 
 [tool.poetry]
 name = "ptars"
-version = "0.0.1-rc1"
+version = "0.0.2-rc2"
 description = "Convert from protobuf to arrow and back in rust"
 authors = ["Tradewell Tech <engineering@tradewelltech.co>"]
 maintainers = ["0x26res <0x26res@gmail.com>"]

{ptars-0.0.1rc1 → ptars-0.0.2rc2}/python/ptars/internal.py

@@ -1,3 +1,7 @@
+import warnings
+
+import pyarrow as pa
+from google._upb._message import Message, MessageMeta
 from google.protobuf.descriptor import Descriptor, FileDescriptor
 from google.protobuf.descriptor_pb2 import FileDescriptorProto
 
@@ -10,16 +14,21 @@ def _file_descriptor_to_bytes(fd: FileDescriptor) -> bytes:
     return file_descriptor.SerializeToString()
 
 
-def get_dependencies(
+def _get_dependencies(
     file_descriptor: FileDescriptor, results: list[FileDescriptor] = None
 ) -> list[FileDescriptor]:
+    """
+    Return list of FileDescriptor that this file depends on, including this one.
+
+    Results are in topological order (least dependent first).
+    """
     if results is None:
         results = []
-    results.append(file_descriptor)
     for dependency in file_descriptor.dependencies:
         if dependency not in results:
-            get_dependencies(dependency, results)
-    return results
+            _get_dependencies(dependency, results)
+    results.append(file_descriptor)
+    return results[::-1]
 
 
 class HandlerPool:
@@ -28,16 +37,30 @@ class HandlerPool:
         self._pool = {}
 
     def get_for_message(self, descriptor: Descriptor) -> MessageHandler:
+        if isinstance(descriptor, MessageMeta):
+            warnings.warn(
+                f"Received {MessageMeta.__name__} instead of {Descriptor.__name__}"
+            )
+            descriptor = descriptor.DESCRIPTOR
+        if not isinstance(descriptor, Descriptor):
+            raise TypeError(f"Expecting {Descriptor.__name__}")
+
         assert isinstance(descriptor, Descriptor)
         try:
-            self._pool[descriptor.full_name]
+            return self._pool[descriptor.full_name]
         except KeyError:
             file_descriptor = descriptor.file
 
-            dependencies = get_dependencies(file_descriptor)
+            dependencies = _get_dependencies(file_descriptor)
             payloads = [_file_descriptor_to_bytes(d) for d in dependencies]
             message_handler = self._proto_cache.create_for_message(
                 "." + descriptor.full_name, payloads
             )
             self._pool[descriptor.full_name] = message_handler
             return message_handler
+
+    def messages_to_record_batch(
+        self, messages: list[Message], descriptor: Descriptor
+    ) -> pa.RecordBatch:
+        handler = self.get_for_message(descriptor)
+        return handler.list_to_record_batch([m.SerializeToString() for m in messages])

{ptars-0.0.1rc1 → ptars-0.0.2rc2}/scripts/protoc.py

@@ -38,7 +38,7 @@ def main():
         "--proto_path={}".format(_GOOGLE_COMMON_PROTOS_ROOT_DIR),
         "--proto_path={}".format(_GRPC_PROTOS_INCLUDE),
         "--proto_path={}".format(_SRC_DIR),
-        "--python_out={}".format(_OUT_DIR),
+        "--python_out={}".format(_ROOT_DIR),
     ] + proto_files
     print(" ".join(proto_args))
     run_protoc(proto_args)

{ptars-0.0.1rc1 → ptars-0.0.2rc2}/src/lib.rs

@@ -565,6 +565,17 @@ impl MessageHandler {
         batch.to_pyarrow(py)
     }
 
+    fn just_convert(&self, values: Vec<Vec<u8>>, _py: Python<'_>) {
+        let _unused: Vec<Box<dyn MessageDyn>> = values
+            .iter()
+            .map(|x| {
+                self.message_descriptor
+                    .parse_from_bytes(x.as_slice())
+                    .unwrap()
+            })
+            .collect();
+    }
+
     fn record_batch_to_array(&self, record_batch: &PyAny, py: Python<'_>) -> PyResult<PyObject> {
         let _arrow_record_batch = RecordBatch::from_pyarrow(record_batch);