palletjack 2.1.2__tar.gz → 2.2.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {palletjack-2.1.2 → palletjack-2.2.1}/PKG-INFO +3 -3
- {palletjack-2.1.2 → palletjack-2.2.1}/README.md +1 -1
- {palletjack-2.1.2 → palletjack-2.2.1}/palletjack/palletjack.cc +2 -6
- {palletjack-2.1.2 → palletjack-2.2.1}/palletjack/palletjack_cython.cpp +1447 -1115
- {palletjack-2.1.2 → palletjack-2.2.1}/palletjack.egg-info/PKG-INFO +3 -3
- palletjack-2.2.1/palletjack.egg-info/requires.txt +1 -0
- {palletjack-2.1.2 → palletjack-2.2.1}/pyproject.toml +3 -3
- palletjack-2.1.2/palletjack.egg-info/requires.txt +0 -1
- {palletjack-2.1.2 → palletjack-2.2.1}/LICENSE +0 -0
- {palletjack-2.1.2 → palletjack-2.2.1}/palletjack/__init__.py +0 -0
- {palletjack-2.1.2 → palletjack-2.2.1}/palletjack/cpalletjack.pxd +0 -0
- {palletjack-2.1.2 → palletjack-2.2.1}/palletjack/palletjack.h +0 -0
- {palletjack-2.1.2 → palletjack-2.2.1}/palletjack/palletjack_cython.pyx +0 -0
- {palletjack-2.1.2 → palletjack-2.2.1}/palletjack/parquet_types_palletjack.cpp +0 -0
- {palletjack-2.1.2 → palletjack-2.2.1}/palletjack/parquet_types_palletjack.h +0 -0
- {palletjack-2.1.2 → palletjack-2.2.1}/palletjack/windows_fixup.h +0 -0
- {palletjack-2.1.2 → palletjack-2.2.1}/palletjack.egg-info/SOURCES.txt +0 -0
- {palletjack-2.1.2 → palletjack-2.2.1}/palletjack.egg-info/dependency_links.txt +0 -0
- {palletjack-2.1.2 → palletjack-2.2.1}/palletjack.egg-info/not-zip-safe +0 -0
- {palletjack-2.1.2 → palletjack-2.2.1}/palletjack.egg-info/top_level.txt +0 -0
- {palletjack-2.1.2 → palletjack-2.2.1}/setup.cfg +0 -0
- {palletjack-2.1.2 → palletjack-2.2.1}/setup.py +0 -0
- {palletjack-2.1.2 → palletjack-2.2.1}/test/test_palletjack.py +0 -0
- {palletjack-2.1.2 → palletjack-2.2.1}/test/test_readme.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: palletjack
|
|
3
|
-
Version: 2.1
|
|
3
|
+
Version: 2.2.1
|
|
4
4
|
Summary: Faster parquet metadata reading
|
|
5
5
|
Author-email: Marcin Krystianc <marcin.krystianc@gmail.com>
|
|
6
6
|
Project-URL: Homepage, https://github.com/marcin-krystianc/PalletJack
|
|
@@ -11,7 +11,7 @@ Classifier: Operating System :: OS Independent
|
|
|
11
11
|
Requires-Python: >=3.9
|
|
12
12
|
Description-Content-Type: text/markdown
|
|
13
13
|
License-File: LICENSE
|
|
14
|
-
Requires-Dist: pyarrow~=
|
|
14
|
+
Requires-Dist: pyarrow~=16.0
|
|
15
15
|
|
|
16
16
|
# PalletJack
|
|
17
17
|
PalletJack was created as a workaround for apache/arrow#38149. The standard parquet reader is not efficient for files with numerous columns and row groups, as it requires parsing the entire metadata section each time the file is opened. The size of this metadata section is proportional to the number of columns and row groups in the file.
|
|
@@ -25,7 +25,7 @@ PalletJack reduces the amount of metadata bytes that need to be read and decoded
|
|
|
25
25
|
|
|
26
26
|
## Required:
|
|
27
27
|
|
|
28
|
-
- pyarrow ~=
|
|
28
|
+
- pyarrow ~= 16.0
|
|
29
29
|
|
|
30
30
|
PalletJack operates on top of pyarrow, making it an essential requirement for both building and using PalletJack. While our source package is compatible with recent versions of pyarrow, the binary distribution package specifically requires the latest major version of pyarrow.
|
|
31
31
|
|
|
@@ -10,7 +10,7 @@ PalletJack reduces the amount of metadata bytes that need to be read and decoded
|
|
|
10
10
|
|
|
11
11
|
## Required:
|
|
12
12
|
|
|
13
|
-
- pyarrow ~=
|
|
13
|
+
- pyarrow ~= 16.0
|
|
14
14
|
|
|
15
15
|
PalletJack operates on top of pyarrow, making it an essential requirement for both building and using PalletJack. While our source package is compatible with recent versions of pyarrow, the binary distribution package specifically requires the latest major version of pyarrow.
|
|
16
16
|
|
|
@@ -87,18 +87,13 @@ using ThriftBuffer = apache::thrift::transport::TMemoryBuffer;
|
|
|
87
87
|
|
|
88
88
|
std::shared_ptr<ThriftBuffer> CreateReadOnlyMemoryBuffer(uint8_t *buf, uint32_t len)
|
|
89
89
|
{
|
|
90
|
-
#if PARQUET_THRIFT_VERSION_MAJOR > 0 || PARQUET_THRIFT_VERSION_MINOR >= 14
|
|
91
90
|
auto conf = std::make_shared<apache::thrift::TConfiguration>();
|
|
92
91
|
conf->setMaxMessageSize(std::numeric_limits<int>::max());
|
|
93
92
|
return std::make_shared<ThriftBuffer>(buf, len, ThriftBuffer::OBSERVE, conf);
|
|
94
|
-
#else
|
|
95
|
-
return std::make_shared<ThriftBuffer>(buf, len);
|
|
96
|
-
#endif
|
|
97
93
|
}
|
|
98
94
|
|
|
99
|
-
template <class T>
|
|
100
95
|
void DeserializeUnencryptedMessage(const uint8_t *buf, uint32_t *len,
|
|
101
|
-
|
|
96
|
+
palletjack::parquet::FileMetaData *deserialized_msg)
|
|
102
97
|
{
|
|
103
98
|
// Deserialize msg bytes into c++ thrift msg using memory transport.
|
|
104
99
|
auto tmem_transport = CreateReadOnlyMemoryBuffer(const_cast<uint8_t *>(buf), *len);
|
|
@@ -107,6 +102,7 @@ void DeserializeUnencryptedMessage(const uint8_t *buf, uint32_t *len,
|
|
|
107
102
|
tproto_factory.setStringSizeLimit(kDefaultThriftStringSizeLimit);
|
|
108
103
|
tproto_factory.setContainerSizeLimit(kDefaultThriftContainerSizeLimit);
|
|
109
104
|
auto tproto = tproto_factory.getProtocol(tmem_transport);
|
|
105
|
+
|
|
110
106
|
try
|
|
111
107
|
{
|
|
112
108
|
deserialized_msg->read(tproto.get());
|