hgraph-persistence 0.8.20__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- hgraph_persistence-0.8.20/.gitignore +7 -0
- hgraph_persistence-0.8.20/CHANGELOG.md +36 -0
- hgraph_persistence-0.8.20/CMakeLists.txt +297 -0
- hgraph_persistence-0.8.20/LICENSE +21 -0
- hgraph_persistence-0.8.20/PKG-INFO +49 -0
- hgraph_persistence-0.8.20/README.md +36 -0
- hgraph_persistence-0.8.20/cmake/hgraph-persistenceConfig.cmake.in +24 -0
- hgraph_persistence-0.8.20/cmake/hgraph_persistence_arrow.cmake +254 -0
- hgraph_persistence-0.8.20/include/hgraph/persistence/export.h +23 -0
- hgraph_persistence-0.8.20/include/hgraph/persistence/frame_store.h +135 -0
- hgraph_persistence-0.8.20/include/hgraph/persistence/object_store.h +140 -0
- hgraph_persistence-0.8.20/include/hgraph/persistence/recording_options.h +120 -0
- hgraph_persistence-0.8.20/include/hgraph/persistence/recording_store.h +113 -0
- hgraph_persistence-0.8.20/include/hgraph/persistence/store_location.h +116 -0
- hgraph_persistence-0.8.20/include/hgraph/persistence/value_codec.h +179 -0
- hgraph_persistence-0.8.20/include/hgraph/persistence/value_store.h +177 -0
- hgraph_persistence-0.8.20/pyproject.toml +51 -0
- hgraph_persistence-0.8.20/python/hgraph_persistence/__init__.py +123 -0
- hgraph_persistence-0.8.20/python/hgraph_persistence/compat.py +412 -0
- hgraph_persistence-0.8.20/python/tests/conftest.py +13 -0
- hgraph_persistence-0.8.20/python/tests/test_api_persistence.py +207 -0
- hgraph_persistence-0.8.20/python/tests/test_data_frame_record_replay.py +310 -0
- hgraph_persistence-0.8.20/python/tests/test_data_frame_replay_window.py +75 -0
- hgraph_persistence-0.8.20/python/tests/test_distribution_audit.py +59 -0
- hgraph_persistence-0.8.20/python/tests/test_global_state_persistence.py +62 -0
- hgraph_persistence-0.8.20/python/tests/test_native_table_recording.py +509 -0
- hgraph_persistence-0.8.20/python/tests/test_packaging.py +136 -0
- hgraph_persistence-0.8.20/python/tests/test_polars_persistence.py +90 -0
- hgraph_persistence-0.8.20/python/tests/test_ported_data_frame_record_replay.py +126 -0
- hgraph_persistence-0.8.20/python/tests/test_python_frame_store.py +253 -0
- hgraph_persistence-0.8.20/src/frame_store.cpp +518 -0
- hgraph_persistence-0.8.20/src/impl/object_store_common.h +64 -0
- hgraph_persistence-0.8.20/src/impl/object_store_impl.h +13 -0
- hgraph_persistence-0.8.20/src/impl/object_store_local.cpp +646 -0
- hgraph_persistence-0.8.20/src/impl/object_store_memory.cpp +123 -0
- hgraph_persistence-0.8.20/src/impl/object_store_s3.cpp +542 -0
- hgraph_persistence-0.8.20/src/impl/s3_options.cpp +88 -0
- hgraph_persistence-0.8.20/src/impl/s3_options.h +23 -0
- hgraph_persistence-0.8.20/src/object_store.cpp +145 -0
- hgraph_persistence-0.8.20/src/python_module.cpp +301 -0
- hgraph_persistence-0.8.20/src/record_replay_frame_impl.cpp +27 -0
- hgraph_persistence-0.8.20/src/record_replay_frame_impl.h +861 -0
- hgraph_persistence-0.8.20/src/recording_store.cpp +225 -0
- hgraph_persistence-0.8.20/src/registration.cpp +49 -0
- hgraph_persistence-0.8.20/src/store_location.cpp +209 -0
- hgraph_persistence-0.8.20/src/value_codec.cpp +335 -0
- hgraph_persistence-0.8.20/src/value_store.cpp +188 -0
- hgraph_persistence-0.8.20/test_package/CMakeLists.txt +31 -0
- hgraph_persistence-0.8.20/test_package/main.cpp +194 -0
- hgraph_persistence-0.8.20/tests/CMakeLists.txt +49 -0
- hgraph_persistence-0.8.20/tests/registry_test_listener.cpp +55 -0
- hgraph_persistence-0.8.20/tests/test_component_frame.cpp +399 -0
- hgraph_persistence-0.8.20/tests/test_frame_store.cpp +563 -0
- hgraph_persistence-0.8.20/tests/test_object_store.cpp +552 -0
- hgraph_persistence-0.8.20/tests/test_record_replay_frame.cpp +759 -0
- hgraph_persistence-0.8.20/tests/test_record_replay_partitioned.cpp +767 -0
- hgraph_persistence-0.8.20/tests/test_replay_const.cpp +109 -0
- hgraph_persistence-0.8.20/tests/test_table_persisted.cpp +413 -0
- hgraph_persistence-0.8.20/tests/test_value_store.cpp +510 -0
- hgraph_persistence-0.8.20/tools/audit_distribution.py +155 -0
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
## Unreleased
|
|
4
|
+
|
|
5
|
+
- Persistence wheels now publish a shared native SDK carrying the pinned curl
|
|
6
|
+
implementation. Downstream extension wheels can consume S3 persistence
|
|
7
|
+
without relinking against an older or ABI-incompatible system CURL/TLS SDK.
|
|
8
|
+
|
|
9
|
+
- Added the reusable C++ `ObjectStore` contract required by RFC 0023 and RFC
|
|
10
|
+
0026: atomic immutable objects, typed absence and backend failures, ordered
|
|
11
|
+
prefix paging, and conditional references across memory, local, and S3
|
|
12
|
+
strategies. Immutable local/S3 `FrameStore` publication now uses the same
|
|
13
|
+
atomic backend path.
|
|
14
|
+
|
|
15
|
+
- Extracted from hgraph core (RFC 0025, checkpoint 4): FrameStore and
|
|
16
|
+
all store backends, the frame-backed record/replay/compare overloads,
|
|
17
|
+
the segmented-recording protocol, the durable replay_const read, and
|
|
18
|
+
the DataFrameStorage Python compatibility surface.
|
|
19
|
+
- Checkpoint 5: registered the durable overloads from the extension's own
|
|
20
|
+
keyed installer; added the missing-extension diagnostics; the `dataframe`
|
|
21
|
+
extra now installs this distribution; the 0.5 override-registry
|
|
22
|
+
translation moved here behind core's wiring-adapter seam.
|
|
23
|
+
- Checkpoint 5 completion (audit 2026-08-19):
|
|
24
|
+
- `RecordAsOf` / `RecordRemoves` are now extension-owned. The C++ enums
|
|
25
|
+
live in `hgraph/persistence/recording_options.h`; the Python spellings
|
|
26
|
+
are exported from `hgraph_persistence` and associated with the native
|
|
27
|
+
scalars through the keyed installer. `hgraph.RecordAsOf` /
|
|
28
|
+
`hgraph.RecordRemoves` remain deprecated aliases until checkpoint 8.
|
|
29
|
+
- The `:data_frame:` recording path and override registry
|
|
30
|
+
(`set_data_frame_record_path`, `set_data_frame_overrides`,
|
|
31
|
+
`get_data_frame_record_overrides`) moved here from core, beside the
|
|
32
|
+
translation that reads them. `hgraph.adaptors.data_frame` re-exports
|
|
33
|
+
them lazily, so released import sites are unchanged.
|
|
34
|
+
- Parquet and S3 detection, linkage and feature macros are this
|
|
35
|
+
extension's alone: it no longer inherits a parent build's answers, and
|
|
36
|
+
its macros are `HGRAPH_PERSISTENCE_WITH_PARQUET` / `_WITH_S3`.
|
|
@@ -0,0 +1,297 @@
|
|
|
1
|
+
cmake_minimum_required(VERSION 3.25)
|
|
2
|
+
|
|
3
|
+
project(hgraph_persistence VERSION 0.8.0 LANGUAGES C CXX)
|
|
4
|
+
|
|
5
|
+
include(GNUInstallDirs)
|
|
6
|
+
include(CMakePackageConfigHelpers)
|
|
7
|
+
include(CTest)
|
|
8
|
+
include(FetchContent)
|
|
9
|
+
|
|
10
|
+
option(HGRAPH_PERSISTENCE_BUILD_PYTHON "Build the optional Python bridge module" OFF)
|
|
11
|
+
option(HGRAPH_PERSISTENCE_WARNINGS_AS_ERRORS
|
|
12
|
+
"Treat hgraph-persistence compiler warnings as errors"
|
|
13
|
+
${HGRAPH_WARNINGS_AS_ERRORS})
|
|
14
|
+
option(HGRAPH_PERSISTENCE_FETCH_CURL
|
|
15
|
+
"Fetch the pinned curl release when a suitable package is unavailable"
|
|
16
|
+
ON)
|
|
17
|
+
option(HGRAPH_PERSISTENCE_FORCE_FETCH_CURL
|
|
18
|
+
"Use the pinned curl release even when a system package is available"
|
|
19
|
+
OFF)
|
|
20
|
+
|
|
21
|
+
if(HGRAPH_PERSISTENCE_BUILD_PYTHON)
|
|
22
|
+
find_package(Python 3.12 COMPONENTS
|
|
23
|
+
Interpreter Development.Module Development.SABIModule REQUIRED)
|
|
24
|
+
endif()
|
|
25
|
+
|
|
26
|
+
if(NOT TARGET hgraph::core)
|
|
27
|
+
find_package(hgraph CONFIG REQUIRED)
|
|
28
|
+
endif()
|
|
29
|
+
|
|
30
|
+
include(cmake/hgraph_persistence_arrow.cmake)
|
|
31
|
+
if(HGRAPH_PERSISTENCE_BUILD_PYTHON)
|
|
32
|
+
hgraph_persistence_find_arrow(PREFER_PYARROW)
|
|
33
|
+
else()
|
|
34
|
+
hgraph_persistence_find_arrow()
|
|
35
|
+
endif()
|
|
36
|
+
|
|
37
|
+
# Arrow deliberately exposes only ordinary S3 filesystem writes. The durable
|
|
38
|
+
# object-store contract needs real If-None-Match/If-Match requests, so its
|
|
39
|
+
# persistence-private S3 strategy uses libcurl's AWS Signature V4 support.
|
|
40
|
+
if(HGRAPH_PERSISTENCE_WITH_S3)
|
|
41
|
+
set(HGRAPH_PERSISTENCE_CURL_TARGET "")
|
|
42
|
+
if(TARGET libcurl_static)
|
|
43
|
+
# A parent such as hgraph-web may already own a pinned curl build.
|
|
44
|
+
set(HGRAPH_PERSISTENCE_CURL_TARGET libcurl_static)
|
|
45
|
+
elseif(NOT HGRAPH_PERSISTENCE_FORCE_FETCH_CURL)
|
|
46
|
+
find_package(CURL 7.75 QUIET)
|
|
47
|
+
if(CURL_FOUND AND TARGET CURL::libcurl)
|
|
48
|
+
set(HGRAPH_PERSISTENCE_CURL_TARGET CURL::libcurl)
|
|
49
|
+
endif()
|
|
50
|
+
endif()
|
|
51
|
+
if(NOT HGRAPH_PERSISTENCE_CURL_TARGET)
|
|
52
|
+
if(NOT HGRAPH_PERSISTENCE_FETCH_CURL)
|
|
53
|
+
message(FATAL_ERROR
|
|
54
|
+
"curl >= 7.75 was not found; install its CMake package or enable "
|
|
55
|
+
"HGRAPH_PERSISTENCE_FETCH_CURL")
|
|
56
|
+
endif()
|
|
57
|
+
|
|
58
|
+
set(_hgraph_persistence_saved_build_shared_libs ${BUILD_SHARED_LIBS})
|
|
59
|
+
set(_hgraph_persistence_saved_build_static_libs ${BUILD_STATIC_LIBS})
|
|
60
|
+
set(_hgraph_persistence_saved_build_testing ${BUILD_TESTING})
|
|
61
|
+
set(_hgraph_persistence_saved_pic ${CMAKE_POSITION_INDEPENDENT_CODE})
|
|
62
|
+
set(BUILD_SHARED_LIBS OFF)
|
|
63
|
+
set(BUILD_STATIC_LIBS ON)
|
|
64
|
+
set(BUILD_TESTING OFF)
|
|
65
|
+
set(CMAKE_POSITION_INDEPENDENT_CODE ON)
|
|
66
|
+
|
|
67
|
+
set(BUILD_CURL_EXE OFF CACHE BOOL "" FORCE)
|
|
68
|
+
set(BUILD_EXAMPLES OFF CACHE BOOL "" FORCE)
|
|
69
|
+
# A static installed hgraph-persistence target still needs libcurl at
|
|
70
|
+
# the final consumer link. Install the pinned curl SDK beside it so a
|
|
71
|
+
# wheel or ordinary SDK install is transitively complete; shared
|
|
72
|
+
# persistence builds carry curl themselves and do not need the SDK.
|
|
73
|
+
if(NOT _hgraph_persistence_saved_build_shared_libs)
|
|
74
|
+
set(_hgraph_persistence_saved_install_component
|
|
75
|
+
"${CMAKE_INSTALL_DEFAULT_COMPONENT_NAME}")
|
|
76
|
+
set(CMAKE_INSTALL_DEFAULT_COMPONENT_NAME Development)
|
|
77
|
+
set(CURL_ENABLE_EXPORT_TARGET ON CACHE BOOL "" FORCE)
|
|
78
|
+
set(CURL_DISABLE_INSTALL OFF CACHE BOOL "" FORCE)
|
|
79
|
+
else()
|
|
80
|
+
set(CURL_ENABLE_EXPORT_TARGET OFF CACHE BOOL "" FORCE)
|
|
81
|
+
set(CURL_DISABLE_INSTALL ON CACHE BOOL "" FORCE)
|
|
82
|
+
endif()
|
|
83
|
+
set(BUILD_LIBCURL_DOCS OFF CACHE BOOL "" FORCE)
|
|
84
|
+
set(BUILD_MISC_DOCS OFF CACHE BOOL "" FORCE)
|
|
85
|
+
set(ENABLE_CURL_MANUAL OFF CACHE BOOL "" FORCE)
|
|
86
|
+
set(USE_NGHTTP2 OFF CACHE BOOL "" FORCE)
|
|
87
|
+
set(CURL_USE_LIBPSL OFF CACHE BOOL "" FORCE)
|
|
88
|
+
set(CURL_USE_LIBSSH2 OFF CACHE BOOL "" FORCE)
|
|
89
|
+
set(CURL_BROTLI OFF CACHE BOOL "" FORCE)
|
|
90
|
+
set(CURL_ZSTD OFF CACHE BOOL "" FORCE)
|
|
91
|
+
set(CURL_ZLIB OFF CACHE BOOL "" FORCE)
|
|
92
|
+
set(USE_LIBIDN2 OFF CACHE BOOL "" FORCE)
|
|
93
|
+
set(CURL_USE_GSSAPI OFF CACHE BOOL "" FORCE)
|
|
94
|
+
set(CURL_DISABLE_LDAP ON CACHE BOOL "" FORCE)
|
|
95
|
+
if(WIN32)
|
|
96
|
+
set(CURL_USE_OPENSSL OFF CACHE BOOL "" FORCE)
|
|
97
|
+
set(CURL_USE_SCHANNEL ON CACHE BOOL "" FORCE)
|
|
98
|
+
set(CURL_USE_SECTRANSP OFF CACHE BOOL "" FORCE)
|
|
99
|
+
elseif(APPLE)
|
|
100
|
+
set(CURL_USE_OPENSSL OFF CACHE BOOL "" FORCE)
|
|
101
|
+
set(CURL_USE_SCHANNEL OFF CACHE BOOL "" FORCE)
|
|
102
|
+
set(CURL_USE_SECTRANSP ON CACHE BOOL "" FORCE)
|
|
103
|
+
else()
|
|
104
|
+
find_package(OpenSSL REQUIRED)
|
|
105
|
+
set(CURL_USE_OPENSSL ON CACHE BOOL "" FORCE)
|
|
106
|
+
set(CURL_USE_SCHANNEL OFF CACHE BOOL "" FORCE)
|
|
107
|
+
set(CURL_USE_SECTRANSP OFF CACHE BOOL "" FORCE)
|
|
108
|
+
endif()
|
|
109
|
+
FetchContent_Declare(hgraph_persistence_curl
|
|
110
|
+
GIT_REPOSITORY https://github.com/curl/curl.git
|
|
111
|
+
GIT_TAG curl-8_11_1
|
|
112
|
+
GIT_SHALLOW TRUE
|
|
113
|
+
)
|
|
114
|
+
FetchContent_MakeAvailable(hgraph_persistence_curl)
|
|
115
|
+
if(TARGET libcurl_static)
|
|
116
|
+
set(HGRAPH_PERSISTENCE_CURL_TARGET libcurl_static)
|
|
117
|
+
elseif(TARGET libcurl)
|
|
118
|
+
set(HGRAPH_PERSISTENCE_CURL_TARGET libcurl)
|
|
119
|
+
else()
|
|
120
|
+
message(FATAL_ERROR "curl did not provide its static library target")
|
|
121
|
+
endif()
|
|
122
|
+
|
|
123
|
+
set(BUILD_SHARED_LIBS ${_hgraph_persistence_saved_build_shared_libs})
|
|
124
|
+
set(BUILD_STATIC_LIBS ${_hgraph_persistence_saved_build_static_libs})
|
|
125
|
+
set(BUILD_TESTING ${_hgraph_persistence_saved_build_testing})
|
|
126
|
+
set(CMAKE_POSITION_INDEPENDENT_CODE ${_hgraph_persistence_saved_pic})
|
|
127
|
+
if(DEFINED _hgraph_persistence_saved_install_component)
|
|
128
|
+
set(CMAKE_INSTALL_DEFAULT_COMPONENT_NAME
|
|
129
|
+
"${_hgraph_persistence_saved_install_component}")
|
|
130
|
+
unset(_hgraph_persistence_saved_install_component)
|
|
131
|
+
endif()
|
|
132
|
+
set(BUILD_TESTING ${_hgraph_persistence_saved_build_testing}
|
|
133
|
+
CACHE BOOL "Build the testing tree." FORCE)
|
|
134
|
+
unset(_hgraph_persistence_saved_build_shared_libs)
|
|
135
|
+
unset(_hgraph_persistence_saved_build_static_libs)
|
|
136
|
+
unset(_hgraph_persistence_saved_build_testing)
|
|
137
|
+
unset(_hgraph_persistence_saved_pic)
|
|
138
|
+
endif()
|
|
139
|
+
endif()
|
|
140
|
+
|
|
141
|
+
if(HGRAPH_PERSISTENCE_BUILD_PYTHON)
|
|
142
|
+
get_target_property(_hgraph_persistence_core_links hgraph::options INTERFACE_LINK_LIBRARIES)
|
|
143
|
+
if("Python::Python" IN_LIST _hgraph_persistence_core_links)
|
|
144
|
+
message(FATAL_ERROR
|
|
145
|
+
"The selected hgraph SDK embeds a specific Python interpreter and "
|
|
146
|
+
"cannot produce an ABI3 hgraph-persistence wheel; use the SDK "
|
|
147
|
+
"installed by an hgraph stable-ABI wheel")
|
|
148
|
+
endif()
|
|
149
|
+
endif()
|
|
150
|
+
|
|
151
|
+
add_library(hgraph_persistence
|
|
152
|
+
src/frame_store.cpp
|
|
153
|
+
src/object_store.cpp
|
|
154
|
+
src/value_codec.cpp
|
|
155
|
+
src/value_store.cpp
|
|
156
|
+
src/store_location.cpp
|
|
157
|
+
src/impl/object_store_memory.cpp
|
|
158
|
+
src/impl/object_store_local.cpp
|
|
159
|
+
src/impl/object_store_s3.cpp
|
|
160
|
+
src/impl/s3_options.cpp
|
|
161
|
+
src/recording_store.cpp
|
|
162
|
+
src/record_replay_frame_impl.cpp
|
|
163
|
+
src/registration.cpp
|
|
164
|
+
)
|
|
165
|
+
add_library(hgraph::persistence ALIAS hgraph_persistence)
|
|
166
|
+
|
|
167
|
+
get_target_property(HGRAPH_PERSISTENCE_LIBRARY_TYPE hgraph_persistence TYPE)
|
|
168
|
+
if(HGRAPH_PERSISTENCE_LIBRARY_TYPE STREQUAL "STATIC_LIBRARY")
|
|
169
|
+
target_compile_definitions(hgraph_persistence PUBLIC HGRAPH_PERSISTENCE_STATIC_DEFINE)
|
|
170
|
+
endif()
|
|
171
|
+
|
|
172
|
+
target_compile_features(hgraph_persistence PUBLIC cxx_std_23)
|
|
173
|
+
target_link_libraries(hgraph_persistence PUBLIC hgraph::core)
|
|
174
|
+
if(TARGET Arrow::arrow_shared)
|
|
175
|
+
target_link_libraries(hgraph_persistence PRIVATE Arrow::arrow_shared)
|
|
176
|
+
else()
|
|
177
|
+
target_link_libraries(hgraph_persistence PRIVATE Arrow::arrow_static)
|
|
178
|
+
endif()
|
|
179
|
+
if(HGRAPH_PERSISTENCE_WITH_PARQUET)
|
|
180
|
+
target_compile_definitions(hgraph_persistence PRIVATE HGRAPH_PERSISTENCE_WITH_PARQUET=1)
|
|
181
|
+
if(TARGET Parquet::parquet_shared)
|
|
182
|
+
target_link_libraries(hgraph_persistence PRIVATE Parquet::parquet_shared)
|
|
183
|
+
elseif(TARGET Parquet::parquet_static)
|
|
184
|
+
target_link_libraries(hgraph_persistence PRIVATE Parquet::parquet_static)
|
|
185
|
+
endif()
|
|
186
|
+
endif()
|
|
187
|
+
if(HGRAPH_PERSISTENCE_WITH_S3)
|
|
188
|
+
target_compile_definitions(hgraph_persistence PRIVATE
|
|
189
|
+
HGRAPH_PERSISTENCE_WITH_S3=1
|
|
190
|
+
HGRAPH_PERSISTENCE_WITH_CURL=1
|
|
191
|
+
)
|
|
192
|
+
# A shared extension carries curl. A static installed SDK consumer links
|
|
193
|
+
# its own CURL::libcurl through hgraph-persistenceConfig.cmake.
|
|
194
|
+
target_link_libraries(hgraph_persistence PRIVATE
|
|
195
|
+
$<BUILD_INTERFACE:${HGRAPH_PERSISTENCE_CURL_TARGET}>)
|
|
196
|
+
endif()
|
|
197
|
+
target_include_directories(hgraph_persistence
|
|
198
|
+
PUBLIC
|
|
199
|
+
$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
|
|
200
|
+
$<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}>
|
|
201
|
+
)
|
|
202
|
+
set_target_properties(hgraph_persistence PROPERTIES
|
|
203
|
+
EXPORT_NAME persistence
|
|
204
|
+
POSITION_INDEPENDENT_CODE ON
|
|
205
|
+
)
|
|
206
|
+
|
|
207
|
+
if(MSVC)
|
|
208
|
+
# Class exports are producer-only; private STL representation is not
|
|
209
|
+
# imported by consumers. Scope these interface diagnostics to this DLL.
|
|
210
|
+
target_compile_options(hgraph_persistence PRIVATE
|
|
211
|
+
/W4 /permissive- /wd4251 /wd4275)
|
|
212
|
+
if(HGRAPH_PERSISTENCE_WARNINGS_AS_ERRORS)
|
|
213
|
+
target_compile_options(hgraph_persistence PRIVATE /WX)
|
|
214
|
+
endif()
|
|
215
|
+
else()
|
|
216
|
+
target_compile_options(hgraph_persistence PRIVATE -Wall -Wextra -Wpedantic)
|
|
217
|
+
if(HGRAPH_PERSISTENCE_WARNINGS_AS_ERRORS)
|
|
218
|
+
target_compile_options(hgraph_persistence PRIVATE -Werror)
|
|
219
|
+
endif()
|
|
220
|
+
endif()
|
|
221
|
+
|
|
222
|
+
if(BUILD_TESTING)
|
|
223
|
+
add_subdirectory(tests)
|
|
224
|
+
endif()
|
|
225
|
+
|
|
226
|
+
if(HGRAPH_PERSISTENCE_BUILD_PYTHON)
|
|
227
|
+
if(NOT COMMAND hgraph_add_python_module OR NOT TARGET hgraph::nanobind)
|
|
228
|
+
message(FATAL_ERROR
|
|
229
|
+
"HGRAPH_PERSISTENCE_BUILD_PYTHON requires a Python-enabled installed hgraph SDK")
|
|
230
|
+
endif()
|
|
231
|
+
# The bridge's RUNPATH locates libhgraph_persistence, but ELF RUNPATH is
|
|
232
|
+
# not inherited when that library resolves its own Arrow/Parquet and core
|
|
233
|
+
# dependencies. Give the shared implementation library an explicit
|
|
234
|
+
# package-relative path as well; otherwise an isolated Linux wheel import
|
|
235
|
+
# depends on some earlier module having loaded libparquet globally.
|
|
236
|
+
if(BUILD_SHARED_LIBS)
|
|
237
|
+
if(APPLE)
|
|
238
|
+
set_target_properties(hgraph_persistence PROPERTIES
|
|
239
|
+
INSTALL_RPATH "@loader_path;@loader_path/../pyarrow")
|
|
240
|
+
elseif(UNIX)
|
|
241
|
+
set_target_properties(hgraph_persistence PROPERTIES
|
|
242
|
+
INSTALL_RPATH "$ORIGIN;$ORIGIN/../pyarrow")
|
|
243
|
+
endif()
|
|
244
|
+
endif()
|
|
245
|
+
hgraph_add_python_module(_hgraph_persistence STABLE_ABI NOMINSIZE src/python_module.cpp)
|
|
246
|
+
target_link_libraries(_hgraph_persistence PRIVATE hgraph::persistence)
|
|
247
|
+
# One more `../` than hgraph_add_python_module's default: the module
|
|
248
|
+
# installs into hgraph_persistence/, and both the core shared runtime
|
|
249
|
+
# (site-packages lib/) and pyarrow's Arrow must resolve from there.
|
|
250
|
+
if(APPLE)
|
|
251
|
+
set_target_properties(_hgraph_persistence PROPERTIES
|
|
252
|
+
INSTALL_RPATH "@loader_path/../${CMAKE_INSTALL_LIBDIR};@loader_path/../pyarrow")
|
|
253
|
+
elseif(UNIX)
|
|
254
|
+
set_target_properties(_hgraph_persistence PROPERTIES
|
|
255
|
+
INSTALL_RPATH "$ORIGIN/../${CMAKE_INSTALL_LIBDIR};$ORIGIN/../pyarrow")
|
|
256
|
+
endif()
|
|
257
|
+
install(TARGETS _hgraph_persistence
|
|
258
|
+
COMPONENT Python
|
|
259
|
+
LIBRARY DESTINATION hgraph_persistence
|
|
260
|
+
RUNTIME DESTINATION hgraph_persistence
|
|
261
|
+
)
|
|
262
|
+
endif()
|
|
263
|
+
|
|
264
|
+
install(TARGETS hgraph_persistence
|
|
265
|
+
EXPORT hgraphPersistenceTargets
|
|
266
|
+
COMPONENT Development
|
|
267
|
+
ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
|
|
268
|
+
LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
|
|
269
|
+
RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
|
|
270
|
+
)
|
|
271
|
+
install(DIRECTORY include/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
|
|
272
|
+
COMPONENT Development FILES_MATCHING PATTERN "*.h")
|
|
273
|
+
|
|
274
|
+
write_basic_package_version_file(
|
|
275
|
+
"${PROJECT_BINARY_DIR}/hgraph-persistenceConfigVersion.cmake"
|
|
276
|
+
VERSION ${PROJECT_VERSION}
|
|
277
|
+
COMPATIBILITY SameMajorVersion
|
|
278
|
+
)
|
|
279
|
+
configure_package_config_file(
|
|
280
|
+
"${CMAKE_CURRENT_SOURCE_DIR}/cmake/hgraph-persistenceConfig.cmake.in"
|
|
281
|
+
"${PROJECT_BINARY_DIR}/hgraph-persistenceConfig.cmake"
|
|
282
|
+
INSTALL_DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/hgraph-persistence
|
|
283
|
+
)
|
|
284
|
+
|
|
285
|
+
install(EXPORT hgraphPersistenceTargets
|
|
286
|
+
FILE hgraphPersistenceTargets.cmake
|
|
287
|
+
NAMESPACE hgraph::
|
|
288
|
+
DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/hgraph-persistence
|
|
289
|
+
COMPONENT Development
|
|
290
|
+
)
|
|
291
|
+
install(FILES
|
|
292
|
+
"${PROJECT_BINARY_DIR}/hgraph-persistenceConfig.cmake"
|
|
293
|
+
"${PROJECT_BINARY_DIR}/hgraph-persistenceConfigVersion.cmake"
|
|
294
|
+
"${CMAKE_CURRENT_SOURCE_DIR}/cmake/hgraph_persistence_arrow.cmake"
|
|
295
|
+
DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/hgraph-persistence
|
|
296
|
+
COMPONENT Development
|
|
297
|
+
)
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2023 Howard Henson
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
Metadata-Version: 2.2
|
|
2
|
+
Name: hgraph-persistence
|
|
3
|
+
Version: 0.8.20
|
|
4
|
+
Summary: Durable record/replay and frame storage for hgraph
|
|
5
|
+
License: MIT
|
|
6
|
+
Project-URL: Homepage, https://github.com/hhenson/hgraph
|
|
7
|
+
Project-URL: Repository, https://github.com/hhenson/hgraph.git
|
|
8
|
+
Requires-Python: >=3.12
|
|
9
|
+
Requires-Dist: hgraph>=0.8.0
|
|
10
|
+
Provides-Extra: test
|
|
11
|
+
Requires-Dist: pytest>=8; extra == "test"
|
|
12
|
+
Description-Content-Type: text/markdown
|
|
13
|
+
|
|
14
|
+
# hgraph-persistence
|
|
15
|
+
|
|
16
|
+
Durable record/replay and frame storage for hgraph (RFC 0025): the
|
|
17
|
+
FrameStore (memory / local filesystem / S3, Arrow IPC / Parquet), the
|
|
18
|
+
frame-backed record/replay/compare overloads for the core operator
|
|
19
|
+
markers, and the DataFrameStorage compatibility surface.
|
|
20
|
+
|
|
21
|
+
The public C++ `ObjectStore` is the durable metadata substrate shared by
|
|
22
|
+
checkpoint/recovery and the versioned dataflow fabric. It provides:
|
|
23
|
+
|
|
24
|
+
- immutable create-if-absent with distinct created, idempotent, and conflict
|
|
25
|
+
outcomes;
|
|
26
|
+
- typed absent reads carrying an opaque version token when present, with
|
|
27
|
+
backend failures reported separately as `ObjectStoreError`;
|
|
28
|
+
- deterministic lexicographic prefix paging;
|
|
29
|
+
- compare/exchange for small named references; and
|
|
30
|
+
- memory, local-filesystem, and S3 strategies behind one owning erased handle.
|
|
31
|
+
|
|
32
|
+
Local immutable publication uses atomic filesystem creation and reference CAS
|
|
33
|
+
uses cross-process file locking plus atomic replacement. S3 conditional writes
|
|
34
|
+
are sent with `If-None-Match` / `If-Match` and AWS Signature V4 through
|
|
35
|
+
libcurl; they are not emulated with a read followed by an unconditional write.
|
|
36
|
+
`FrameStore` immutable local and S3 writes use this same backend publication
|
|
37
|
+
path after serialising the Frame. This requires one additional
|
|
38
|
+
serialized-Frame-sized buffer for an immutable local or S3 write; mutable
|
|
39
|
+
FrameStore writes retain their streaming path.
|
|
40
|
+
|
|
41
|
+
Wheel builds embed a pinned private curl for conditional S3 requests. A shared
|
|
42
|
+
`hgraph::persistence` library carries that implementation itself; consumers of
|
|
43
|
+
a static installed C++ library must provide `CURL::libcurl` version 7.75 or
|
|
44
|
+
newer at the final link.
|
|
45
|
+
|
|
46
|
+
Installing and importing `hgraph_persistence` registers the
|
|
47
|
+
`"hgraph.persistence.frame"` backend with the shared hgraph runtime;
|
|
48
|
+
selecting that backend (`set_record_replay_model`) activates it from
|
|
49
|
+
unchanged `hgraph` imports.
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
# hgraph-persistence
|
|
2
|
+
|
|
3
|
+
Durable record/replay and frame storage for hgraph (RFC 0025): the
|
|
4
|
+
FrameStore (memory / local filesystem / S3, Arrow IPC / Parquet), the
|
|
5
|
+
frame-backed record/replay/compare overloads for the core operator
|
|
6
|
+
markers, and the DataFrameStorage compatibility surface.
|
|
7
|
+
|
|
8
|
+
The public C++ `ObjectStore` is the durable metadata substrate shared by
|
|
9
|
+
checkpoint/recovery and the versioned dataflow fabric. It provides:
|
|
10
|
+
|
|
11
|
+
- immutable create-if-absent with distinct created, idempotent, and conflict
|
|
12
|
+
outcomes;
|
|
13
|
+
- typed absent reads carrying an opaque version token when present, with
|
|
14
|
+
backend failures reported separately as `ObjectStoreError`;
|
|
15
|
+
- deterministic lexicographic prefix paging;
|
|
16
|
+
- compare/exchange for small named references; and
|
|
17
|
+
- memory, local-filesystem, and S3 strategies behind one owning erased handle.
|
|
18
|
+
|
|
19
|
+
Local immutable publication uses atomic filesystem creation and reference CAS
|
|
20
|
+
uses cross-process file locking plus atomic replacement. S3 conditional writes
|
|
21
|
+
are sent with `If-None-Match` / `If-Match` and AWS Signature V4 through
|
|
22
|
+
libcurl; they are not emulated with a read followed by an unconditional write.
|
|
23
|
+
`FrameStore` immutable local and S3 writes use this same backend publication
|
|
24
|
+
path after serialising the Frame. This requires one additional
|
|
25
|
+
serialized-Frame-sized buffer for an immutable local or S3 write; mutable
|
|
26
|
+
FrameStore writes retain their streaming path.
|
|
27
|
+
|
|
28
|
+
Wheel builds embed a pinned private curl for conditional S3 requests. A shared
|
|
29
|
+
`hgraph::persistence` library carries that implementation itself; consumers of
|
|
30
|
+
a static installed C++ library must provide `CURL::libcurl` version 7.75 or
|
|
31
|
+
newer at the final link.
|
|
32
|
+
|
|
33
|
+
Installing and importing `hgraph_persistence` registers the
|
|
34
|
+
`"hgraph.persistence.frame"` backend with the shared hgraph runtime;
|
|
35
|
+
selecting that backend (`set_record_replay_model`) activates it from
|
|
36
|
+
unchanged `hgraph` imports.
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
@PACKAGE_INIT@
|
|
2
|
+
|
|
3
|
+
include(CMakeFindDependencyMacro)
|
|
4
|
+
find_dependency(hgraph CONFIG)
|
|
5
|
+
|
|
6
|
+
include("${CMAKE_CURRENT_LIST_DIR}/hgraph_persistence_arrow.cmake")
|
|
7
|
+
if(TARGET hgraph::nanobind)
|
|
8
|
+
hgraph_persistence_find_arrow(PREFER_PYARROW)
|
|
9
|
+
else()
|
|
10
|
+
hgraph_persistence_find_arrow()
|
|
11
|
+
endif()
|
|
12
|
+
|
|
13
|
+
include("${CMAKE_CURRENT_LIST_DIR}/hgraphPersistenceTargets.cmake")
|
|
14
|
+
|
|
15
|
+
if(@HGRAPH_PERSISTENCE_WITH_S3@)
|
|
16
|
+
get_target_property(_hgraph_persistence_type hgraph::persistence TYPE)
|
|
17
|
+
if(_hgraph_persistence_type STREQUAL "STATIC_LIBRARY")
|
|
18
|
+
find_dependency(CURL 7.75)
|
|
19
|
+
target_link_libraries(hgraph::persistence INTERFACE CURL::libcurl)
|
|
20
|
+
endif()
|
|
21
|
+
unset(_hgraph_persistence_type)
|
|
22
|
+
endif()
|
|
23
|
+
|
|
24
|
+
check_required_components(hgraph-persistence)
|