rocksdb-native 2.2.0 → 2.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (261) hide show
  1. package/binding.c +92 -10
  2. package/index.js +9 -0
  3. package/lib/batch.js +11 -1
  4. package/lib/iterator.js +3 -1
  5. package/lib/snapshot.js +21 -0
  6. package/package.json +1 -1
  7. package/prebuilds/darwin-arm64/rocksdb-native.bare +0 -0
  8. package/prebuilds/darwin-arm64/rocksdb-native.node +0 -0
  9. package/prebuilds/darwin-x64/rocksdb-native.bare +0 -0
  10. package/prebuilds/darwin-x64/rocksdb-native.node +0 -0
  11. package/prebuilds/linux-arm64/rocksdb-native.bare +0 -0
  12. package/prebuilds/linux-arm64/rocksdb-native.node +0 -0
  13. package/prebuilds/linux-x64/rocksdb-native.bare +0 -0
  14. package/prebuilds/linux-x64/rocksdb-native.node +0 -0
  15. package/prebuilds/win32-x64/rocksdb-native.bare +0 -0
  16. package/prebuilds/win32-x64/rocksdb-native.node +0 -0
  17. package/vendor/librocksdb/include/rocksdb.h +38 -4
  18. package/vendor/librocksdb/src/rocksdb.cc +114 -14
  19. package/vendor/librocksdb/vendor/rocksdb/CMakeLists.txt +21 -4
  20. package/vendor/librocksdb/vendor/rocksdb/cache/secondary_cache_adapter.cc +6 -3
  21. package/vendor/librocksdb/vendor/rocksdb/db/arena_wrapped_db_iter.cc +4 -4
  22. package/vendor/librocksdb/vendor/rocksdb/db/arena_wrapped_db_iter.h +4 -2
  23. package/vendor/librocksdb/vendor/rocksdb/db/attribute_group_iterator_impl.cc +20 -0
  24. package/vendor/librocksdb/vendor/rocksdb/db/attribute_group_iterator_impl.h +83 -0
  25. package/vendor/librocksdb/vendor/rocksdb/db/builder.cc +9 -5
  26. package/vendor/librocksdb/vendor/rocksdb/db/builder.h +1 -1
  27. package/vendor/librocksdb/vendor/rocksdb/db/c.cc +231 -6
  28. package/vendor/librocksdb/vendor/rocksdb/db/c_test.c +202 -2
  29. package/vendor/librocksdb/vendor/rocksdb/db/coalescing_iterator.cc +47 -0
  30. package/vendor/librocksdb/vendor/rocksdb/db/coalescing_iterator.h +79 -0
  31. package/vendor/librocksdb/vendor/rocksdb/db/column_family.cc +28 -0
  32. package/vendor/librocksdb/vendor/rocksdb/db/column_family.h +17 -0
  33. package/vendor/librocksdb/vendor/rocksdb/db/compaction/compaction.cc +8 -1
  34. package/vendor/librocksdb/vendor/rocksdb/db/compaction/compaction.h +11 -9
  35. package/vendor/librocksdb/vendor/rocksdb/db/compaction/compaction_iterator.cc +50 -23
  36. package/vendor/librocksdb/vendor/rocksdb/db/compaction/compaction_iterator.h +13 -0
  37. package/vendor/librocksdb/vendor/rocksdb/db/compaction/compaction_job.cc +22 -25
  38. package/vendor/librocksdb/vendor/rocksdb/db/compaction/compaction_job.h +2 -0
  39. package/vendor/librocksdb/vendor/rocksdb/db/compaction/compaction_outputs.cc +8 -1
  40. package/vendor/librocksdb/vendor/rocksdb/db/compaction/compaction_outputs.h +1 -0
  41. package/vendor/librocksdb/vendor/rocksdb/db/compaction/compaction_picker.cc +40 -17
  42. package/vendor/librocksdb/vendor/rocksdb/db/compaction/compaction_picker.h +20 -14
  43. package/vendor/librocksdb/vendor/rocksdb/db/compaction/compaction_picker_level.cc +11 -6
  44. package/vendor/librocksdb/vendor/rocksdb/db/compaction/compaction_picker_universal.cc +77 -24
  45. package/vendor/librocksdb/vendor/rocksdb/db/compaction/compaction_service_job.cc +2 -0
  46. package/vendor/librocksdb/vendor/rocksdb/db/convenience.cc +3 -0
  47. package/vendor/librocksdb/vendor/rocksdb/db/db_filesnapshot.cc +125 -31
  48. package/vendor/librocksdb/vendor/rocksdb/db/db_impl/db_impl.cc +457 -231
  49. package/vendor/librocksdb/vendor/rocksdb/db/db_impl/db_impl.h +172 -73
  50. package/vendor/librocksdb/vendor/rocksdb/db/db_impl/db_impl_compaction_flush.cc +152 -133
  51. package/vendor/librocksdb/vendor/rocksdb/db/db_impl/db_impl_debug.cc +5 -0
  52. package/vendor/librocksdb/vendor/rocksdb/db/db_impl/db_impl_files.cc +58 -52
  53. package/vendor/librocksdb/vendor/rocksdb/db/db_impl/db_impl_follower.cc +348 -0
  54. package/vendor/librocksdb/vendor/rocksdb/db/db_impl/db_impl_follower.h +54 -0
  55. package/vendor/librocksdb/vendor/rocksdb/db/db_impl/db_impl_open.cc +136 -117
  56. package/vendor/librocksdb/vendor/rocksdb/db/db_impl/db_impl_secondary.cc +4 -3
  57. package/vendor/librocksdb/vendor/rocksdb/db/db_impl/db_impl_secondary.h +7 -6
  58. package/vendor/librocksdb/vendor/rocksdb/db/db_impl/db_impl_write.cc +134 -80
  59. package/vendor/librocksdb/vendor/rocksdb/db/db_iter.cc +11 -0
  60. package/vendor/librocksdb/vendor/rocksdb/db/db_test2.cc +1 -1
  61. package/vendor/librocksdb/vendor/rocksdb/db/db_test_util.cc +11 -1
  62. package/vendor/librocksdb/vendor/rocksdb/db/db_test_util.h +11 -7
  63. package/vendor/librocksdb/vendor/rocksdb/db/dbformat.cc +19 -4
  64. package/vendor/librocksdb/vendor/rocksdb/db/dbformat.h +3 -2
  65. package/vendor/librocksdb/vendor/rocksdb/db/error_handler.cc +34 -39
  66. package/vendor/librocksdb/vendor/rocksdb/db/error_handler.h +3 -4
  67. package/vendor/librocksdb/vendor/rocksdb/db/event_helpers.cc +6 -3
  68. package/vendor/librocksdb/vendor/rocksdb/db/experimental.cc +3 -2
  69. package/vendor/librocksdb/vendor/rocksdb/db/external_sst_file_ingestion_job.cc +76 -18
  70. package/vendor/librocksdb/vendor/rocksdb/db/external_sst_file_ingestion_job.h +11 -0
  71. package/vendor/librocksdb/vendor/rocksdb/db/flush_job.cc +37 -5
  72. package/vendor/librocksdb/vendor/rocksdb/db/flush_job.h +14 -0
  73. package/vendor/librocksdb/vendor/rocksdb/db/import_column_family_job.cc +49 -45
  74. package/vendor/librocksdb/vendor/rocksdb/db/internal_stats.cc +60 -1
  75. package/vendor/librocksdb/vendor/rocksdb/db/internal_stats.h +20 -1
  76. package/vendor/librocksdb/vendor/rocksdb/db/log_reader.cc +15 -6
  77. package/vendor/librocksdb/vendor/rocksdb/db/log_writer.cc +59 -10
  78. package/vendor/librocksdb/vendor/rocksdb/db/log_writer.h +8 -0
  79. package/vendor/librocksdb/vendor/rocksdb/db/memtable.cc +24 -40
  80. package/vendor/librocksdb/vendor/rocksdb/db/memtable.h +10 -10
  81. package/vendor/librocksdb/vendor/rocksdb/db/memtable_list.cc +9 -8
  82. package/vendor/librocksdb/vendor/rocksdb/db/multi_cf_iterator_impl.h +296 -0
  83. package/vendor/librocksdb/vendor/rocksdb/db/range_tombstone_fragmenter.h +8 -10
  84. package/vendor/librocksdb/vendor/rocksdb/db/repair.cc +4 -3
  85. package/vendor/librocksdb/vendor/rocksdb/db/seqno_to_time_mapping.cc +30 -0
  86. package/vendor/librocksdb/vendor/rocksdb/db/seqno_to_time_mapping.h +9 -0
  87. package/vendor/librocksdb/vendor/rocksdb/db/table_cache.cc +17 -2
  88. package/vendor/librocksdb/vendor/rocksdb/db/table_cache.h +9 -1
  89. package/vendor/librocksdb/vendor/rocksdb/db/table_properties_collector.h +9 -2
  90. package/vendor/librocksdb/vendor/rocksdb/db/transaction_log_impl.cc +3 -3
  91. package/vendor/librocksdb/vendor/rocksdb/db/transaction_log_impl.h +7 -7
  92. package/vendor/librocksdb/vendor/rocksdb/db/version_edit.cc +0 -1
  93. package/vendor/librocksdb/vendor/rocksdb/db/version_edit_handler.cc +39 -5
  94. package/vendor/librocksdb/vendor/rocksdb/db/version_edit_handler.h +24 -15
  95. package/vendor/librocksdb/vendor/rocksdb/db/version_set.cc +117 -64
  96. package/vendor/librocksdb/vendor/rocksdb/db/version_set.h +27 -10
  97. package/vendor/librocksdb/vendor/rocksdb/db/wal_manager.cc +37 -29
  98. package/vendor/librocksdb/vendor/rocksdb/db/wal_manager.h +6 -5
  99. package/vendor/librocksdb/vendor/rocksdb/db/wide/wide_columns.cc +2 -3
  100. package/vendor/librocksdb/vendor/rocksdb/db/wide/wide_columns_helper.cc +6 -0
  101. package/vendor/librocksdb/vendor/rocksdb/db/write_batch.cc +89 -31
  102. package/vendor/librocksdb/vendor/rocksdb/db/write_thread.cc +53 -5
  103. package/vendor/librocksdb/vendor/rocksdb/db/write_thread.h +36 -4
  104. package/vendor/librocksdb/vendor/rocksdb/env/composite_env_wrapper.h +21 -0
  105. package/vendor/librocksdb/vendor/rocksdb/env/env.cc +15 -0
  106. package/vendor/librocksdb/vendor/rocksdb/env/fs_on_demand.cc +331 -0
  107. package/vendor/librocksdb/vendor/rocksdb/env/fs_on_demand.h +139 -0
  108. package/vendor/librocksdb/vendor/rocksdb/env/io_posix.cc +8 -6
  109. package/vendor/librocksdb/vendor/rocksdb/env/io_posix.h +1 -1
  110. package/vendor/librocksdb/vendor/rocksdb/file/delete_scheduler.cc +130 -27
  111. package/vendor/librocksdb/vendor/rocksdb/file/delete_scheduler.h +61 -8
  112. package/vendor/librocksdb/vendor/rocksdb/file/file_util.cc +25 -4
  113. package/vendor/librocksdb/vendor/rocksdb/file/file_util.h +15 -0
  114. package/vendor/librocksdb/vendor/rocksdb/file/sequence_file_reader.cc +1 -0
  115. package/vendor/librocksdb/vendor/rocksdb/file/sequence_file_reader.h +9 -4
  116. package/vendor/librocksdb/vendor/rocksdb/file/sst_file_manager_impl.cc +18 -0
  117. package/vendor/librocksdb/vendor/rocksdb/file/sst_file_manager_impl.h +31 -4
  118. package/vendor/librocksdb/vendor/rocksdb/file/writable_file_writer.cc +40 -38
  119. package/vendor/librocksdb/vendor/rocksdb/file/writable_file_writer.h +48 -15
  120. package/vendor/librocksdb/vendor/rocksdb/include/rocksdb/advanced_options.h +12 -3
  121. package/vendor/librocksdb/vendor/rocksdb/include/rocksdb/attribute_groups.h +114 -0
  122. package/vendor/librocksdb/vendor/rocksdb/include/rocksdb/c.h +90 -0
  123. package/vendor/librocksdb/vendor/rocksdb/include/rocksdb/cache.h +5 -0
  124. package/vendor/librocksdb/vendor/rocksdb/include/rocksdb/comparator.h +27 -0
  125. package/vendor/librocksdb/vendor/rocksdb/include/rocksdb/db.h +71 -12
  126. package/vendor/librocksdb/vendor/rocksdb/include/rocksdb/env.h +9 -0
  127. package/vendor/librocksdb/vendor/rocksdb/include/rocksdb/experimental.h +5 -0
  128. package/vendor/librocksdb/vendor/rocksdb/include/rocksdb/file_system.h +14 -0
  129. package/vendor/librocksdb/vendor/rocksdb/include/rocksdb/iterator.h +9 -71
  130. package/vendor/librocksdb/vendor/rocksdb/include/rocksdb/iterator_base.h +90 -0
  131. package/vendor/librocksdb/vendor/rocksdb/include/rocksdb/listener.h +21 -0
  132. package/vendor/librocksdb/vendor/rocksdb/include/rocksdb/options.h +125 -12
  133. package/vendor/librocksdb/vendor/rocksdb/include/rocksdb/perf_context.h +1 -1
  134. package/vendor/librocksdb/vendor/rocksdb/include/rocksdb/sst_file_reader.h +11 -1
  135. package/vendor/librocksdb/vendor/rocksdb/include/rocksdb/table.h +6 -6
  136. package/vendor/librocksdb/vendor/rocksdb/include/rocksdb/table_properties.h +19 -0
  137. package/vendor/librocksdb/vendor/rocksdb/include/rocksdb/transaction_log.h +12 -6
  138. package/vendor/librocksdb/vendor/rocksdb/include/rocksdb/types.h +12 -0
  139. package/vendor/librocksdb/vendor/rocksdb/include/rocksdb/universal_compaction.h +31 -0
  140. package/vendor/librocksdb/vendor/rocksdb/include/rocksdb/user_write_callback.h +29 -0
  141. package/vendor/librocksdb/vendor/rocksdb/include/rocksdb/utilities/cache_dump_load.h +4 -0
  142. package/vendor/librocksdb/vendor/rocksdb/include/rocksdb/utilities/checkpoint.h +4 -2
  143. package/vendor/librocksdb/vendor/rocksdb/include/rocksdb/utilities/customizable_util.h +0 -1
  144. package/vendor/librocksdb/vendor/rocksdb/include/rocksdb/utilities/env_mirror.h +1 -1
  145. package/vendor/librocksdb/vendor/rocksdb/include/rocksdb/utilities/ldb_cmd.h +24 -7
  146. package/vendor/librocksdb/vendor/rocksdb/include/rocksdb/utilities/option_change_migration.h +4 -4
  147. package/vendor/librocksdb/vendor/rocksdb/include/rocksdb/utilities/stackable_db.h +24 -5
  148. package/vendor/librocksdb/vendor/rocksdb/include/rocksdb/utilities/table_properties_collectors.h +46 -0
  149. package/vendor/librocksdb/vendor/rocksdb/include/rocksdb/utilities/transaction.h +42 -17
  150. package/vendor/librocksdb/vendor/rocksdb/include/rocksdb/utilities/transaction_db.h +5 -0
  151. package/vendor/librocksdb/vendor/rocksdb/include/rocksdb/utilities/types_util.h +36 -0
  152. package/vendor/librocksdb/vendor/rocksdb/include/rocksdb/utilities/write_batch_with_index.h +71 -3
  153. package/vendor/librocksdb/vendor/rocksdb/include/rocksdb/version.h +2 -2
  154. package/vendor/librocksdb/vendor/rocksdb/include/rocksdb/wide_columns.h +87 -72
  155. package/vendor/librocksdb/vendor/rocksdb/include/rocksdb/write_batch_base.h +1 -1
  156. package/vendor/librocksdb/vendor/rocksdb/memory/memory_allocator.cc +1 -0
  157. package/vendor/librocksdb/vendor/rocksdb/options/cf_options.cc +13 -2
  158. package/vendor/librocksdb/vendor/rocksdb/options/cf_options.h +6 -2
  159. package/vendor/librocksdb/vendor/rocksdb/options/db_options.cc +27 -1
  160. package/vendor/librocksdb/vendor/rocksdb/options/db_options.h +10 -3
  161. package/vendor/librocksdb/vendor/rocksdb/options/options.cc +3 -0
  162. package/vendor/librocksdb/vendor/rocksdb/options/options_helper.cc +1 -0
  163. package/vendor/librocksdb/vendor/rocksdb/port/jemalloc_helper.h +2 -2
  164. package/vendor/librocksdb/vendor/rocksdb/port/stack_trace.cc +1 -0
  165. package/vendor/librocksdb/vendor/rocksdb/port/win/port_win.cc +3 -2
  166. package/vendor/librocksdb/vendor/rocksdb/table/block_based/binary_search_index_reader.cc +1 -2
  167. package/vendor/librocksdb/vendor/rocksdb/table/block_based/block_based_table_builder.cc +47 -31
  168. package/vendor/librocksdb/vendor/rocksdb/table/block_based/block_based_table_factory.cc +15 -0
  169. package/vendor/librocksdb/vendor/rocksdb/table/block_based/block_based_table_iterator.cc +37 -18
  170. package/vendor/librocksdb/vendor/rocksdb/table/block_based/block_based_table_iterator.h +10 -3
  171. package/vendor/librocksdb/vendor/rocksdb/table/block_based/block_based_table_reader.cc +102 -41
  172. package/vendor/librocksdb/vendor/rocksdb/table/block_based/block_based_table_reader.h +15 -7
  173. package/vendor/librocksdb/vendor/rocksdb/table/block_based/block_based_table_reader_impl.h +1 -3
  174. package/vendor/librocksdb/vendor/rocksdb/table/block_based/block_based_table_reader_sync_and_async.h +5 -6
  175. package/vendor/librocksdb/vendor/rocksdb/table/block_based/block_cache.h +31 -0
  176. package/vendor/librocksdb/vendor/rocksdb/table/block_based/block_prefetcher.cc +6 -0
  177. package/vendor/librocksdb/vendor/rocksdb/table/block_based/cachable_entry.h +10 -5
  178. package/vendor/librocksdb/vendor/rocksdb/table/block_based/filter_block.h +34 -28
  179. package/vendor/librocksdb/vendor/rocksdb/table/block_based/filter_block_reader_common.cc +17 -11
  180. package/vendor/librocksdb/vendor/rocksdb/table/block_based/filter_block_reader_common.h +5 -2
  181. package/vendor/librocksdb/vendor/rocksdb/table/block_based/filter_policy.cc +12 -3
  182. package/vendor/librocksdb/vendor/rocksdb/table/block_based/full_filter_block.cc +37 -30
  183. package/vendor/librocksdb/vendor/rocksdb/table/block_based/full_filter_block.h +11 -13
  184. package/vendor/librocksdb/vendor/rocksdb/table/block_based/hash_index_reader.cc +1 -2
  185. package/vendor/librocksdb/vendor/rocksdb/table/block_based/index_builder.cc +62 -53
  186. package/vendor/librocksdb/vendor/rocksdb/table/block_based/index_builder.h +60 -38
  187. package/vendor/librocksdb/vendor/rocksdb/table/block_based/index_reader_common.cc +14 -9
  188. package/vendor/librocksdb/vendor/rocksdb/table/block_based/index_reader_common.h +4 -1
  189. package/vendor/librocksdb/vendor/rocksdb/table/block_based/partitioned_filter_block.cc +135 -94
  190. package/vendor/librocksdb/vendor/rocksdb/table/block_based/partitioned_filter_block.h +52 -46
  191. package/vendor/librocksdb/vendor/rocksdb/table/block_based/partitioned_index_reader.cc +51 -13
  192. package/vendor/librocksdb/vendor/rocksdb/table/block_based/partitioned_index_reader.h +2 -0
  193. package/vendor/librocksdb/vendor/rocksdb/table/block_based/uncompression_dict_reader.cc +3 -11
  194. package/vendor/librocksdb/vendor/rocksdb/table/block_based/uncompression_dict_reader.h +2 -3
  195. package/vendor/librocksdb/vendor/rocksdb/table/block_fetcher.cc +8 -10
  196. package/vendor/librocksdb/vendor/rocksdb/table/block_fetcher.h +2 -1
  197. package/vendor/librocksdb/vendor/rocksdb/table/compaction_merging_iterator.cc +9 -10
  198. package/vendor/librocksdb/vendor/rocksdb/table/compaction_merging_iterator.h +3 -2
  199. package/vendor/librocksdb/vendor/rocksdb/table/format.cc +1 -2
  200. package/vendor/librocksdb/vendor/rocksdb/table/iterator.cc +4 -0
  201. package/vendor/librocksdb/vendor/rocksdb/table/merging_iterator.cc +18 -13
  202. package/vendor/librocksdb/vendor/rocksdb/table/merging_iterator.h +5 -3
  203. package/vendor/librocksdb/vendor/rocksdb/table/meta_blocks.cc +18 -4
  204. package/vendor/librocksdb/vendor/rocksdb/table/meta_blocks.h +4 -0
  205. package/vendor/librocksdb/vendor/rocksdb/table/plain/plain_table_builder.cc +2 -2
  206. package/vendor/librocksdb/vendor/rocksdb/table/sst_file_dumper.cc +6 -6
  207. package/vendor/librocksdb/vendor/rocksdb/table/sst_file_reader.cc +24 -2
  208. package/vendor/librocksdb/vendor/rocksdb/table/sst_file_writer_collectors.h +3 -1
  209. package/vendor/librocksdb/vendor/rocksdb/table/table_builder.h +8 -7
  210. package/vendor/librocksdb/vendor/rocksdb/table/table_iterator.h +69 -0
  211. package/vendor/librocksdb/vendor/rocksdb/table/table_reader.h +9 -0
  212. package/vendor/librocksdb/vendor/rocksdb/test_util/testutil.cc +25 -0
  213. package/vendor/librocksdb/vendor/rocksdb/test_util/testutil.h +12 -0
  214. package/vendor/librocksdb/vendor/rocksdb/tools/db_bench_tool.cc +32 -0
  215. package/vendor/librocksdb/vendor/rocksdb/tools/ldb_cmd.cc +618 -124
  216. package/vendor/librocksdb/vendor/rocksdb/tools/ldb_cmd_impl.h +19 -1
  217. package/vendor/librocksdb/vendor/rocksdb/tools/ldb_tool.cc +9 -0
  218. package/vendor/librocksdb/vendor/rocksdb/util/aligned_storage.h +24 -0
  219. package/vendor/librocksdb/vendor/rocksdb/util/autovector.h +4 -0
  220. package/vendor/librocksdb/vendor/rocksdb/util/comparator.cc +12 -0
  221. package/vendor/librocksdb/vendor/rocksdb/util/filter_bench.cc +1 -1
  222. package/vendor/librocksdb/vendor/rocksdb/util/random.cc +2 -1
  223. package/vendor/librocksdb/vendor/rocksdb/util/stderr_logger.cc +3 -4
  224. package/vendor/librocksdb/vendor/rocksdb/util/stderr_logger.h +1 -1
  225. package/vendor/librocksdb/vendor/rocksdb/util/udt_util.cc +33 -0
  226. package/vendor/librocksdb/vendor/rocksdb/util/udt_util.h +7 -0
  227. package/vendor/librocksdb/vendor/rocksdb/util/write_batch_util.h +5 -0
  228. package/vendor/librocksdb/vendor/rocksdb/util/xxhash.h +36 -29
  229. package/vendor/librocksdb/vendor/rocksdb/utilities/blob_db/blob_db_impl.h +3 -0
  230. package/vendor/librocksdb/vendor/rocksdb/utilities/blob_db/blob_db_impl_filesnapshot.cc +20 -0
  231. package/vendor/librocksdb/vendor/rocksdb/utilities/cache_dump_load_impl.cc +29 -9
  232. package/vendor/librocksdb/vendor/rocksdb/utilities/cache_dump_load_impl.h +14 -3
  233. package/vendor/librocksdb/vendor/rocksdb/utilities/debug.cc +16 -4
  234. package/vendor/librocksdb/vendor/rocksdb/utilities/fault_injection_fs.cc +677 -248
  235. package/vendor/librocksdb/vendor/rocksdb/utilities/fault_injection_fs.h +325 -158
  236. package/vendor/librocksdb/vendor/rocksdb/utilities/option_change_migration/option_change_migration.cc +1 -8
  237. package/vendor/librocksdb/vendor/rocksdb/utilities/table_properties_collectors/compact_for_tiering_collector.cc +144 -0
  238. package/vendor/librocksdb/vendor/rocksdb/utilities/table_properties_collectors/compact_for_tiering_collector.h +45 -0
  239. package/vendor/librocksdb/vendor/rocksdb/utilities/table_properties_collectors/compact_on_deletion_collector.cc +12 -0
  240. package/vendor/librocksdb/vendor/rocksdb/utilities/transactions/lock/range/range_tree/lib/portability/toku_time.h +1 -1
  241. package/vendor/librocksdb/vendor/rocksdb/utilities/transactions/lock/range/range_tree/lib/util/growable_array.h +3 -3
  242. package/vendor/librocksdb/vendor/rocksdb/utilities/transactions/pessimistic_transaction.cc +116 -20
  243. package/vendor/librocksdb/vendor/rocksdb/utilities/transactions/pessimistic_transaction.h +33 -1
  244. package/vendor/librocksdb/vendor/rocksdb/utilities/transactions/pessimistic_transaction_db.cc +78 -13
  245. package/vendor/librocksdb/vendor/rocksdb/utilities/transactions/pessimistic_transaction_db.h +33 -1
  246. package/vendor/librocksdb/vendor/rocksdb/utilities/transactions/transaction_base.cc +106 -7
  247. package/vendor/librocksdb/vendor/rocksdb/utilities/transactions/transaction_base.h +68 -10
  248. package/vendor/librocksdb/vendor/rocksdb/utilities/transactions/transaction_test.h +7 -3
  249. package/vendor/librocksdb/vendor/rocksdb/utilities/transactions/transaction_util.cc +8 -5
  250. package/vendor/librocksdb/vendor/rocksdb/utilities/transactions/transaction_util.h +7 -4
  251. package/vendor/librocksdb/vendor/rocksdb/utilities/transactions/write_prepared_txn.cc +18 -12
  252. package/vendor/librocksdb/vendor/rocksdb/utilities/transactions/write_prepared_txn_db.cc +4 -4
  253. package/vendor/librocksdb/vendor/rocksdb/utilities/transactions/write_prepared_txn_db.h +17 -0
  254. package/vendor/librocksdb/vendor/rocksdb/utilities/transactions/write_unprepared_txn.cc +11 -9
  255. package/vendor/librocksdb/vendor/rocksdb/utilities/transactions/write_unprepared_txn_db.cc +2 -1
  256. package/vendor/librocksdb/vendor/rocksdb/utilities/types_util.cc +88 -0
  257. package/vendor/librocksdb/vendor/rocksdb/utilities/write_batch_with_index/write_batch_with_index.cc +313 -14
  258. package/vendor/librocksdb/vendor/rocksdb/utilities/write_batch_with_index/write_batch_with_index_internal.cc +7 -0
  259. package/vendor/librocksdb/vendor/rocksdb/utilities/write_batch_with_index/write_batch_with_index_internal.h +1 -1
  260. package/vendor/librocksdb/vendor/rocksdb/db/multi_cf_iterator.cc +0 -102
  261. package/vendor/librocksdb/vendor/rocksdb/db/multi_cf_iterator.h +0 -159
@@ -45,6 +45,7 @@
45
45
  #include "util/file_checksum_helper.h"
46
46
  #include "util/stderr_logger.h"
47
47
  #include "util/string_util.h"
48
+ #include "util/write_batch_util.h"
48
49
  #include "utilities/blob_db/blob_dump_tool.h"
49
50
  #include "utilities/merge_operators.h"
50
51
  #include "utilities/ttl/db_ttl_impl.h"
@@ -59,6 +60,7 @@ const std::string LDBCommand::ARG_FS_URI = "fs_uri";
59
60
  const std::string LDBCommand::ARG_DB = "db";
60
61
  const std::string LDBCommand::ARG_PATH = "path";
61
62
  const std::string LDBCommand::ARG_SECONDARY_PATH = "secondary_path";
63
+ const std::string LDBCommand::ARG_LEADER_PATH = "leader_path";
62
64
  const std::string LDBCommand::ARG_HEX = "hex";
63
65
  const std::string LDBCommand::ARG_KEY_HEX = "key_hex";
64
66
  const std::string LDBCommand::ARG_VALUE_HEX = "value_hex";
@@ -107,6 +109,7 @@ const std::string LDBCommand::ARG_PREPOPULATE_BLOB_CACHE =
107
109
  const std::string LDBCommand::ARG_DECODE_BLOB_INDEX = "decode_blob_index";
108
110
  const std::string LDBCommand::ARG_DUMP_UNCOMPRESSED_BLOBS =
109
111
  "dump_uncompressed_blobs";
112
+ const std::string LDBCommand::ARG_READ_TIMESTAMP = "read_timestamp";
110
113
 
111
114
  const char* LDBCommand::DELIM = " ==> ";
112
115
 
@@ -114,6 +117,7 @@ namespace {
114
117
 
115
118
  void DumpWalFile(Options options, std::string wal_file, bool print_header,
116
119
  bool print_values, bool is_write_committed,
120
+ const std::map<uint32_t, const Comparator*>& ucmps,
117
121
  LDBCommandExecuteResult* exec_state);
118
122
 
119
123
  void DumpSstFile(Options options, std::string filename, bool output_hex,
@@ -122,6 +126,9 @@ void DumpSstFile(Options options, std::string filename, bool output_hex,
122
126
 
123
127
  void DumpBlobFile(const std::string& filename, bool is_key_hex,
124
128
  bool is_value_hex, bool dump_uncompressed_blobs);
129
+
130
+ Status EncodeUserProvidedTimestamp(const std::string& user_timestamp,
131
+ std::string* ts_buf);
125
132
  } // namespace
126
133
 
127
134
  LDBCommand* LDBCommand::InitFromCmdLineArgs(
@@ -136,6 +143,32 @@ LDBCommand* LDBCommand::InitFromCmdLineArgs(
136
143
  SelectCommand);
137
144
  }
138
145
 
146
+ void LDBCommand::ParseSingleParam(const std::string& param,
147
+ ParsedParams& parsed_params,
148
+ std::vector<std::string>& cmd_tokens) {
149
+ const std::string OPTION_PREFIX = "--";
150
+
151
+ if (param[0] == '-' && param[1] == '-') {
152
+ std::vector<std::string> splits = StringSplit(param, '=');
153
+ // --option_name=option_value
154
+ if (splits.size() == 2) {
155
+ std::string optionKey = splits[0].substr(OPTION_PREFIX.size());
156
+ parsed_params.option_map[optionKey] = splits[1];
157
+ } else if (splits.size() == 1) {
158
+ // --flag_name
159
+ std::string optionKey = splits[0].substr(OPTION_PREFIX.size());
160
+ parsed_params.flags.push_back(optionKey);
161
+ } else {
162
+ // --option_name=option_value, option_value contains '='
163
+ std::string optionKey = splits[0].substr(OPTION_PREFIX.size());
164
+ parsed_params.option_map[optionKey] =
165
+ param.substr(splits[0].length() + 1);
166
+ }
167
+ } else {
168
+ cmd_tokens.push_back(param);
169
+ }
170
+ }
171
+
139
172
  /**
140
173
  * Parse the command-line arguments and create the appropriate LDBCommand2
141
174
  * instance.
@@ -162,28 +195,8 @@ LDBCommand* LDBCommand::InitFromCmdLineArgs(
162
195
  // and their parameters. For eg: put key1 value1 go into this vector.
163
196
  std::vector<std::string> cmdTokens;
164
197
 
165
- const std::string OPTION_PREFIX = "--";
166
-
167
198
  for (const auto& arg : args) {
168
- if (arg[0] == '-' && arg[1] == '-') {
169
- std::vector<std::string> splits = StringSplit(arg, '=');
170
- // --option_name=option_value
171
- if (splits.size() == 2) {
172
- std::string optionKey = splits[0].substr(OPTION_PREFIX.size());
173
- parsed_params.option_map[optionKey] = splits[1];
174
- } else if (splits.size() == 1) {
175
- // --flag_name
176
- std::string optionKey = splits[0].substr(OPTION_PREFIX.size());
177
- parsed_params.flags.push_back(optionKey);
178
- } else {
179
- // --option_name=option_value, option_value contains '='
180
- std::string optionKey = splits[0].substr(OPTION_PREFIX.size());
181
- parsed_params.option_map[optionKey] =
182
- arg.substr(splits[0].length() + 1);
183
- }
184
- } else {
185
- cmdTokens.push_back(arg);
186
- }
199
+ ParseSingleParam(arg, parsed_params, cmdTokens);
187
200
  }
188
201
 
189
202
  if (cmdTokens.size() < 1) {
@@ -214,6 +227,10 @@ LDBCommand* LDBCommand::SelectCommand(const ParsedParams& parsed_params) {
214
227
  } else if (parsed_params.cmd == GetEntityCommand::Name()) {
215
228
  return new GetEntityCommand(parsed_params.cmd_params,
216
229
  parsed_params.option_map, parsed_params.flags);
230
+ } else if (parsed_params.cmd == MultiGetEntityCommand::Name()) {
231
+ return new MultiGetEntityCommand(parsed_params.cmd_params,
232
+ parsed_params.option_map,
233
+ parsed_params.flags);
217
234
  } else if (parsed_params.cmd == PutCommand::Name()) {
218
235
  return new PutCommand(parsed_params.cmd_params, parsed_params.option_map,
219
236
  parsed_params.flags);
@@ -422,6 +439,12 @@ LDBCommand::LDBCommand(const std::map<std::string, std::string>& options,
422
439
  secondary_path_ = itr->second;
423
440
  }
424
441
 
442
+ itr = options.find(ARG_LEADER_PATH);
443
+ leader_path_ = "";
444
+ if (itr != options.end()) {
445
+ leader_path_ = itr->second;
446
+ }
447
+
425
448
  is_key_hex_ = IsKeyHex(options, flags);
426
449
  is_value_hex_ = IsValueHex(options, flags);
427
450
  is_db_ttl_ = IsFlagPresent(flags, ARG_TTL);
@@ -454,9 +477,9 @@ void LDBCommand::OpenDB() {
454
477
  exec_state_ = LDBCommandExecuteResult::Failed(
455
478
  "ldb doesn't support TTL DB with multiple column families");
456
479
  }
457
- if (!secondary_path_.empty()) {
480
+ if (!secondary_path_.empty() || !leader_path_.empty()) {
458
481
  exec_state_ = LDBCommandExecuteResult::Failed(
459
- "Open as secondary is not supported for TTL DB yet.");
482
+ "Open as secondary or follower is not supported for TTL DB yet.");
460
483
  }
461
484
  if (is_read_only_) {
462
485
  st = DBWithTTL::Open(options_, db_path_, &db_ttl_, 0, true);
@@ -465,7 +488,11 @@ void LDBCommand::OpenDB() {
465
488
  }
466
489
  db_ = db_ttl_;
467
490
  } else {
468
- if (is_read_only_ && secondary_path_.empty()) {
491
+ if (!secondary_path_.empty() && !leader_path_.empty()) {
492
+ exec_state_ = LDBCommandExecuteResult::Failed(
493
+ "Cannot provide both secondary and leader paths");
494
+ }
495
+ if (is_read_only_ && secondary_path_.empty() && leader_path_.empty()) {
469
496
  if (column_families_.empty()) {
470
497
  st = DB::OpenForReadOnly(options_, db_path_, &db_);
471
498
  } else {
@@ -474,18 +501,27 @@ void LDBCommand::OpenDB() {
474
501
  }
475
502
  } else {
476
503
  if (column_families_.empty()) {
477
- if (secondary_path_.empty()) {
504
+ if (secondary_path_.empty() && leader_path_.empty()) {
478
505
  st = DB::Open(options_, db_path_, &db_);
479
- } else {
506
+ } else if (!secondary_path_.empty()) {
480
507
  st = DB::OpenAsSecondary(options_, db_path_, secondary_path_, &db_);
508
+ } else {
509
+ std::unique_ptr<DB> dbptr;
510
+ st = DB::OpenAsFollower(options_, db_path_, leader_path_, &dbptr);
511
+ db_ = dbptr.release();
481
512
  }
482
513
  } else {
483
- if (secondary_path_.empty()) {
514
+ if (secondary_path_.empty() && leader_path_.empty()) {
484
515
  st = DB::Open(options_, db_path_, column_families_, &handles_opened,
485
516
  &db_);
486
- } else {
517
+ } else if (!secondary_path_.empty()) {
487
518
  st = DB::OpenAsSecondary(options_, db_path_, secondary_path_,
488
519
  column_families_, &handles_opened, &db_);
520
+ } else {
521
+ std::unique_ptr<DB> dbptr;
522
+ st = DB::OpenAsFollower(options_, db_path_, leader_path_,
523
+ column_families_, &handles_opened, &dbptr);
524
+ db_ = dbptr.release();
489
525
  }
490
526
  }
491
527
  }
@@ -498,6 +534,7 @@ void LDBCommand::OpenDB() {
498
534
  bool found_cf_name = false;
499
535
  for (size_t i = 0; i < handles_opened.size(); i++) {
500
536
  cf_handles_[column_families_[i].name] = handles_opened[i];
537
+ ucmps_[handles_opened[i]->GetID()] = handles_opened[i]->GetComparator();
501
538
  if (column_family_name_ == column_families_[i].name) {
502
539
  found_cf_name = true;
503
540
  }
@@ -507,6 +544,8 @@ void LDBCommand::OpenDB() {
507
544
  "Non-existing column family " + column_family_name_);
508
545
  CloseDB();
509
546
  }
547
+ ColumnFamilyHandle* default_cf = db_->DefaultColumnFamily();
548
+ ucmps_[default_cf->GetID()] = default_cf->GetComparator();
510
549
  } else {
511
550
  // We successfully opened DB in single column family mode.
512
551
  assert(column_families_.empty());
@@ -515,6 +554,8 @@ void LDBCommand::OpenDB() {
515
554
  "Non-existing column family " + column_family_name_);
516
555
  CloseDB();
517
556
  }
557
+ ColumnFamilyHandle* default_cf = db_->DefaultColumnFamily();
558
+ ucmps_[default_cf->GetID()] = default_cf->GetComparator();
518
559
  }
519
560
  }
520
561
 
@@ -549,6 +590,7 @@ std::vector<std::string> LDBCommand::BuildCmdLineOptions(
549
590
  ARG_FS_URI,
550
591
  ARG_DB,
551
592
  ARG_SECONDARY_PATH,
593
+ ARG_LEADER_PATH,
552
594
  ARG_BLOOM_BITS,
553
595
  ARG_BLOCK_SIZE,
554
596
  ARG_AUTO_COMPACTION,
@@ -718,6 +760,42 @@ bool LDBCommand::ParseCompressionTypeOption(
718
760
  return false;
719
761
  }
720
762
 
763
+ Status LDBCommand::MaybePopulateReadTimestamp(ColumnFamilyHandle* cfh,
764
+ ReadOptions& ropts,
765
+ Slice* read_timestamp) {
766
+ const size_t ts_sz = cfh->GetComparator()->timestamp_size();
767
+
768
+ auto iter = option_map_.find(ARG_READ_TIMESTAMP);
769
+ if (iter == option_map_.end()) {
770
+ if (ts_sz == 0) {
771
+ return Status::OK();
772
+ }
773
+ return Status::InvalidArgument(
774
+ "column family enables user-defined timestamp while --read_timestamp "
775
+ "is not provided.");
776
+ }
777
+ if (iter->second.empty()) {
778
+ if (ts_sz == 0) {
779
+ return Status::OK();
780
+ }
781
+ return Status::InvalidArgument(
782
+ "column family enables user-defined timestamp while --read_timestamp "
783
+ "is empty.");
784
+ }
785
+ if (ts_sz == 0) {
786
+ return Status::InvalidArgument(
787
+ "column family does not enable user-defined timestamps while "
788
+ "--read_timestamp is provided.");
789
+ }
790
+ Status s = EncodeUserProvidedTimestamp(iter->second, &read_timestamp_);
791
+ if (!s.ok()) {
792
+ return s;
793
+ }
794
+ *read_timestamp = read_timestamp_;
795
+ ropts.timestamp = read_timestamp;
796
+ return Status::OK();
797
+ }
798
+
721
799
  void LDBCommand::OverrideBaseOptions() {
722
800
  options_.create_if_missing = false;
723
801
 
@@ -765,7 +843,10 @@ void LDBCommand::OverrideBaseCFOptions(ColumnFamilyOptions* cf_opts) {
765
843
  }
766
844
  }
767
845
 
768
- if (options_.comparator != nullptr) {
846
+ // Default comparator is BytewiseComparator, so only when it's not, it
847
+ // means user has a command line override.
848
+ if (options_.comparator != nullptr &&
849
+ options_.comparator != BytewiseComparator()) {
769
850
  cf_opts->comparator = options_.comparator;
770
851
  }
771
852
 
@@ -1098,27 +1179,36 @@ std::string LDBCommand::StringToHex(const std::string& str) {
1098
1179
  }
1099
1180
 
1100
1181
  std::string LDBCommand::PrintKeyValue(const std::string& key,
1182
+ const std::string& timestamp,
1101
1183
  const std::string& value, bool is_key_hex,
1102
- bool is_value_hex) {
1184
+ bool is_value_hex,
1185
+ const Comparator* ucmp) {
1103
1186
  std::string result;
1104
1187
  result.append(is_key_hex ? StringToHex(key) : key);
1188
+ if (!timestamp.empty()) {
1189
+ result.append("|timestamp:");
1190
+ result.append(ucmp->TimestampToString(timestamp));
1191
+ }
1105
1192
  result.append(DELIM);
1106
1193
  result.append(is_value_hex ? StringToHex(value) : value);
1107
1194
  return result;
1108
1195
  }
1109
1196
 
1110
1197
  std::string LDBCommand::PrintKeyValue(const std::string& key,
1111
- const std::string& value, bool is_hex) {
1112
- return PrintKeyValue(key, value, is_hex, is_hex);
1198
+ const std::string& timestamp,
1199
+ const std::string& value, bool is_hex,
1200
+ const Comparator* ucmp) {
1201
+ return PrintKeyValue(key, timestamp, value, is_hex, is_hex, ucmp);
1113
1202
  }
1114
1203
 
1115
1204
  std::string LDBCommand::PrintKeyValueOrWideColumns(
1116
- const Slice& key, const Slice& value, const WideColumns& wide_columns,
1117
- bool is_key_hex, bool is_value_hex) {
1205
+ const Slice& key, const Slice& timestamp, const Slice& value,
1206
+ const WideColumns& wide_columns, bool is_key_hex, bool is_value_hex,
1207
+ const Comparator* ucmp) {
1118
1208
  if (wide_columns.empty() ||
1119
1209
  WideColumnsHelper::HasDefaultColumnOnly(wide_columns)) {
1120
- return PrintKeyValue(key.ToString(), value.ToString(), is_key_hex,
1121
- is_value_hex);
1210
+ return PrintKeyValue(key.ToString(), timestamp.ToString(), value.ToString(),
1211
+ is_key_hex, is_value_hex, ucmp);
1122
1212
  }
1123
1213
  /*
1124
1214
  // Sample plaintext output (first column is kDefaultWideColumnName)
@@ -1129,9 +1219,10 @@ std::string LDBCommand::PrintKeyValueOrWideColumns(
1129
1219
  */
1130
1220
  std::ostringstream oss;
1131
1221
  WideColumnsHelper::DumpWideColumns(wide_columns, oss, is_value_hex);
1132
- return PrintKeyValue(key.ToString(), oss.str().c_str(), is_key_hex,
1133
- false); // is_value_hex_ is already honored in oss.
1134
- // avoid double-hexing it.
1222
+ return PrintKeyValue(key.ToString(), timestamp.ToString(), oss.str().c_str(),
1223
+ is_key_hex, false,
1224
+ ucmp); // is_value_hex_ is already honored in oss.
1225
+ // avoid double-hexing it.
1135
1226
  }
1136
1227
 
1137
1228
  std::string LDBCommand::HelpRangeCmdArgs() {
@@ -1881,10 +1972,12 @@ void InternalDumpCommand::DoCommand() {
1881
1972
  assert(GetExecuteState().IsFailed());
1882
1973
  return;
1883
1974
  }
1884
-
1975
+ ColumnFamilyHandle* cfh = GetCfHandle();
1976
+ const Comparator* ucmp = cfh->GetComparator();
1977
+ size_t ts_sz = ucmp->timestamp_size();
1885
1978
  if (print_stats_) {
1886
1979
  std::string stats;
1887
- if (db_->GetProperty(GetCfHandle(), "rocksdb.stats", &stats)) {
1980
+ if (db_->GetProperty(cfh, "rocksdb.stats", &stats)) {
1888
1981
  fprintf(stdout, "%s\n", stats.c_str());
1889
1982
  }
1890
1983
  }
@@ -1906,7 +1999,11 @@ void InternalDumpCommand::DoCommand() {
1906
1999
  for (auto& key_version : key_versions) {
1907
2000
  ValueType value_type = static_cast<ValueType>(key_version.type);
1908
2001
  InternalKey ikey(key_version.user_key, key_version.sequence, value_type);
1909
- if (has_to_ && ikey.user_key() == to_) {
2002
+ Slice user_key_without_ts = ikey.user_key();
2003
+ if (ts_sz > 0) {
2004
+ user_key_without_ts.remove_suffix(ts_sz);
2005
+ }
2006
+ if (has_to_ && ucmp->Compare(user_key_without_ts, to_) == 0) {
1910
2007
  // GetAllKeyVersions() includes keys with user key `to_`, but idump has
1911
2008
  // traditionally excluded such keys.
1912
2009
  break;
@@ -1942,7 +2039,7 @@ void InternalDumpCommand::DoCommand() {
1942
2039
  }
1943
2040
 
1944
2041
  if (!count_only_ && !count_delim_) {
1945
- std::string key = ikey.DebugString(is_key_hex_);
2042
+ std::string key = ikey.DebugString(is_key_hex_, ucmp);
1946
2043
  Slice value(key_version.value);
1947
2044
  if (!decode_blob_index_ || value_type != kTypeBlobIndex) {
1948
2045
  if (value_type == kTypeWideColumnEntity) {
@@ -2118,7 +2215,7 @@ void DBDumperCommand::DoCommand() {
2118
2215
  // TODO(myabandeh): allow configuring is_write_commited
2119
2216
  DumpWalFile(options_, path_, /* print_header_ */ true,
2120
2217
  /* print_values_ */ true, true /* is_write_commited */,
2121
- &exec_state_);
2218
+ ucmps_, &exec_state_);
2122
2219
  break;
2123
2220
  case kTableFile:
2124
2221
  DumpSstFile(options_, path_, is_key_hex_, /* show_properties */ true,
@@ -2158,8 +2255,16 @@ void DBDumperCommand::DoDumpCommand() {
2158
2255
 
2159
2256
  // Setup key iterator
2160
2257
  ReadOptions scan_read_opts;
2258
+ Slice read_timestamp;
2259
+ ColumnFamilyHandle* cfh = GetCfHandle();
2260
+ const Comparator* ucmp = cfh->GetComparator();
2261
+ size_t ts_sz = ucmp->timestamp_size();
2262
+ if (ucmp->timestamp_size() > 0) {
2263
+ read_timestamp = ucmp->GetMaxTimestamp();
2264
+ scan_read_opts.timestamp = &read_timestamp;
2265
+ }
2161
2266
  scan_read_opts.total_order_seek = true;
2162
- Iterator* iter = db_->NewIterator(scan_read_opts, GetCfHandle());
2267
+ Iterator* iter = db_->NewIterator(scan_read_opts, cfh);
2163
2268
  Status st = iter->status();
2164
2269
  if (!st.ok()) {
2165
2270
  exec_state_ =
@@ -2214,7 +2319,7 @@ void DBDumperCommand::DoDumpCommand() {
2214
2319
  for (; iter->Valid(); iter->Next()) {
2215
2320
  int rawtime = 0;
2216
2321
  // If end marker was specified, we stop before it
2217
- if (!null_to_ && (iter->key().ToString() >= to_)) {
2322
+ if (!null_to_ && ucmp->Compare(iter->key(), to_) >= 0) {
2218
2323
  break;
2219
2324
  }
2220
2325
  // Terminate if maximum number of keys have been dumped
@@ -2268,11 +2373,14 @@ void DBDumperCommand::DoDumpCommand() {
2268
2373
  // (TODO) TTL Iterator does not support wide columns yet.
2269
2374
  std::string str =
2270
2375
  is_db_ttl_
2271
- ? PrintKeyValue(iter->key().ToString(), iter->value().ToString(),
2272
- is_key_hex_, is_value_hex_)
2273
- : PrintKeyValueOrWideColumns(iter->key(), iter->value(),
2274
- iter->columns(), is_key_hex_,
2275
- is_value_hex_);
2376
+ ? PrintKeyValue(iter->key().ToString(),
2377
+ ts_sz == 0 ? "" : iter->timestamp().ToString(),
2378
+ iter->value().ToString(), is_key_hex_,
2379
+ is_value_hex_, ucmp)
2380
+ : PrintKeyValueOrWideColumns(
2381
+ iter->key(), ts_sz == 0 ? "" : iter->timestamp().ToString(),
2382
+ iter->value(), iter->columns(), is_key_hex_, is_value_hex_,
2383
+ ucmp);
2276
2384
  fprintf(stdout, "%s\n", str.c_str());
2277
2385
  }
2278
2386
  }
@@ -2593,14 +2701,16 @@ struct StdErrReporter : public log::Reader::Reporter {
2593
2701
  class InMemoryHandler : public WriteBatch::Handler {
2594
2702
  public:
2595
2703
  InMemoryHandler(std::stringstream& row, bool print_values,
2596
- bool write_after_commit = false)
2704
+ bool write_after_commit,
2705
+ const std::map<uint32_t, const Comparator*>& ucmps)
2597
2706
  : Handler(),
2598
2707
  row_(row),
2599
2708
  print_values_(print_values),
2600
- write_after_commit_(write_after_commit) {}
2709
+ write_after_commit_(write_after_commit),
2710
+ ucmps_(ucmps) {}
2601
2711
 
2602
- void commonPutMerge(const Slice& key, const Slice& value) {
2603
- std::string k = LDBCommand::StringToHex(key.ToString());
2712
+ void commonPutMerge(uint32_t cf, const Slice& key, const Slice& value) {
2713
+ std::string k = PrintKey(cf, key);
2604
2714
  if (print_values_) {
2605
2715
  std::string v = LDBCommand::StringToHex(value.ToString());
2606
2716
  row_ << k << " : ";
@@ -2612,23 +2722,29 @@ class InMemoryHandler : public WriteBatch::Handler {
2612
2722
 
2613
2723
  Status PutCF(uint32_t cf, const Slice& key, const Slice& value) override {
2614
2724
  row_ << "PUT(" << cf << ") : ";
2615
- commonPutMerge(key, value);
2725
+ commonPutMerge(cf, key, value);
2616
2726
  return Status::OK();
2617
2727
  }
2618
2728
 
2619
2729
  Status PutEntityCF(uint32_t cf, const Slice& key,
2620
2730
  const Slice& value) override {
2621
- row_ << "PUT_ENTITY(" << cf << ") : ";
2622
- std::string k = LDBCommand::StringToHex(key.ToString());
2731
+ row_ << "PUT_ENTITY(" << cf << ") : " << PrintKey(cf, key);
2623
2732
  if (print_values_) {
2624
- return WideColumnsHelper::DumpSliceAsWideColumns(value, row_, true);
2733
+ row_ << " : ";
2734
+ const Status s =
2735
+ WideColumnsHelper::DumpSliceAsWideColumns(value, row_, true);
2736
+ if (!s.ok()) {
2737
+ return s;
2738
+ }
2625
2739
  }
2740
+
2741
+ row_ << ' ';
2626
2742
  return Status::OK();
2627
2743
  }
2628
2744
 
2629
2745
  Status MergeCF(uint32_t cf, const Slice& key, const Slice& value) override {
2630
2746
  row_ << "MERGE(" << cf << ") : ";
2631
- commonPutMerge(key, value);
2747
+ commonPutMerge(cf, key, value);
2632
2748
  return Status::OK();
2633
2749
  }
2634
2750
 
@@ -2639,21 +2755,21 @@ class InMemoryHandler : public WriteBatch::Handler {
2639
2755
 
2640
2756
  Status DeleteCF(uint32_t cf, const Slice& key) override {
2641
2757
  row_ << "DELETE(" << cf << ") : ";
2642
- row_ << LDBCommand::StringToHex(key.ToString()) << " ";
2758
+ row_ << PrintKey(cf, key) << " ";
2643
2759
  return Status::OK();
2644
2760
  }
2645
2761
 
2646
2762
  Status SingleDeleteCF(uint32_t cf, const Slice& key) override {
2647
2763
  row_ << "SINGLE_DELETE(" << cf << ") : ";
2648
- row_ << LDBCommand::StringToHex(key.ToString()) << " ";
2764
+ row_ << PrintKey(cf, key) << " ";
2649
2765
  return Status::OK();
2650
2766
  }
2651
2767
 
2652
2768
  Status DeleteRangeCF(uint32_t cf, const Slice& begin_key,
2653
2769
  const Slice& end_key) override {
2654
2770
  row_ << "DELETE_RANGE(" << cf << ") : ";
2655
- row_ << LDBCommand::StringToHex(begin_key.ToString()) << " ";
2656
- row_ << LDBCommand::StringToHex(end_key.ToString()) << " ";
2771
+ row_ << PrintKey(cf, begin_key) << " ";
2772
+ row_ << PrintKey(cf, end_key) << " ";
2657
2773
  return Status::OK();
2658
2774
  }
2659
2775
 
@@ -2698,13 +2814,37 @@ class InMemoryHandler : public WriteBatch::Handler {
2698
2814
  }
2699
2815
 
2700
2816
  private:
2817
+ std::string PrintKey(uint32_t cf, const Slice& key) {
2818
+ auto ucmp_iter = ucmps_.find(cf);
2819
+ if (ucmp_iter == ucmps_.end()) {
2820
+ // Fallback to default print slice as hex
2821
+ return LDBCommand::StringToHex(key.ToString());
2822
+ }
2823
+ size_t ts_sz = ucmp_iter->second->timestamp_size();
2824
+ if (ts_sz == 0) {
2825
+ return LDBCommand::StringToHex(key.ToString());
2826
+ } else {
2827
+ // This could happen if there is corruption or undetected comparator
2828
+ // change.
2829
+ if (key.size() < ts_sz) {
2830
+ return "CORRUPT KEY";
2831
+ }
2832
+ Slice user_key_without_ts = key;
2833
+ user_key_without_ts.remove_suffix(ts_sz);
2834
+ Slice ts = Slice(key.data() + key.size() - ts_sz, ts_sz);
2835
+ return LDBCommand::StringToHex(user_key_without_ts.ToString()) +
2836
+ "|timestamp:" + ucmp_iter->second->TimestampToString(ts);
2837
+ }
2838
+ }
2701
2839
  std::stringstream& row_;
2702
2840
  bool print_values_;
2703
2841
  bool write_after_commit_;
2842
+ const std::map<uint32_t, const Comparator*> ucmps_;
2704
2843
  };
2705
2844
 
2706
2845
  void DumpWalFile(Options options, std::string wal_file, bool print_header,
2707
2846
  bool print_values, bool is_write_committed,
2847
+ const std::map<uint32_t, const Comparator*>& ucmps,
2708
2848
  LDBCommandExecuteResult* exec_state) {
2709
2849
  const auto& fs = options.env->GetFileSystem();
2710
2850
  FileOptions soptions(options);
@@ -2725,6 +2865,12 @@ void DumpWalFile(Options options, std::string wal_file, bool print_header,
2725
2865
  uint64_t log_number;
2726
2866
  FileType type;
2727
2867
 
2868
+ // Comparators are available and will be used for formatting user key if DB
2869
+ // is opened for this dump wal operation.
2870
+ UnorderedMap<uint32_t, size_t> running_ts_sz;
2871
+ for (const auto& [cf_id, ucmp] : ucmps) {
2872
+ running_ts_sz.emplace(cf_id, ucmp->timestamp_size());
2873
+ }
2728
2874
  // we need the log number, but ParseFilename expects dbname/NNN.log.
2729
2875
  std::string sanitized = wal_file;
2730
2876
  size_t lastslash = sanitized.rfind('/');
@@ -2737,6 +2883,7 @@ void DumpWalFile(Options options, std::string wal_file, bool print_header,
2737
2883
  }
2738
2884
  log::Reader reader(options.info_log, std::move(wal_file_reader), &reporter,
2739
2885
  true /* checksum */, log_number);
2886
+ std::unordered_set<uint32_t> encountered_cf_ids;
2740
2887
  std::string scratch;
2741
2888
  WriteBatch batch;
2742
2889
  Slice record;
@@ -2765,11 +2912,51 @@ void DumpWalFile(Options options, std::string wal_file, bool print_header,
2765
2912
  }
2766
2913
  break;
2767
2914
  }
2915
+ const UnorderedMap<uint32_t, size_t> recorded_ts_sz =
2916
+ reader.GetRecordedTimestampSize();
2917
+ if (!running_ts_sz.empty()) {
2918
+ status = HandleWriteBatchTimestampSizeDifference(
2919
+ &batch, running_ts_sz, recorded_ts_sz,
2920
+ TimestampSizeConsistencyMode::kVerifyConsistency,
2921
+ /*new_batch=*/nullptr);
2922
+ if (!status.ok()) {
2923
+ std::stringstream oss;
2924
+ oss << "Format for user keys in WAL file is inconsistent with the "
2925
+ "comparator used to open the DB. Timestamp size recorded in "
2926
+ "WAL vs specified by "
2927
+ "comparator: {";
2928
+ bool first_cf = true;
2929
+ for (const auto& [cf_id, ts_sz] : running_ts_sz) {
2930
+ if (first_cf) {
2931
+ first_cf = false;
2932
+ } else {
2933
+ oss << ", ";
2934
+ }
2935
+ auto record_ts_iter = recorded_ts_sz.find(cf_id);
2936
+ size_t ts_sz_in_wal = (record_ts_iter == recorded_ts_sz.end())
2937
+ ? 0
2938
+ : record_ts_iter->second;
2939
+ oss << "(cf_id: " << cf_id << ", [recorded: " << ts_sz_in_wal
2940
+ << ", comparator: " << ts_sz << "])";
2941
+ }
2942
+ oss << "}";
2943
+ if (exec_state) {
2944
+ *exec_state = LDBCommandExecuteResult::Failed(oss.str());
2945
+ } else {
2946
+ std::cerr << oss.str() << std::endl;
2947
+ }
2948
+ break;
2949
+ }
2950
+ }
2768
2951
  row << WriteBatchInternal::Sequence(&batch) << ",";
2769
2952
  row << WriteBatchInternal::Count(&batch) << ",";
2770
2953
  row << WriteBatchInternal::ByteSize(&batch) << ",";
2771
2954
  row << reader.LastRecordOffset() << ",";
2772
- InMemoryHandler handler(row, print_values, is_write_committed);
2955
+ ColumnFamilyCollector cf_collector;
2956
+ status = batch.Iterate(&cf_collector);
2957
+ auto cf_ids = cf_collector.column_families();
2958
+ encountered_cf_ids.insert(cf_ids.begin(), cf_ids.end());
2959
+ InMemoryHandler handler(row, print_values, is_write_committed, ucmps);
2773
2960
  status = batch.Iterate(&handler);
2774
2961
  if (!status.ok()) {
2775
2962
  if (exec_state) {
@@ -2784,6 +2971,29 @@ void DumpWalFile(Options options, std::string wal_file, bool print_header,
2784
2971
  }
2785
2972
  std::cout << row.str();
2786
2973
  }
2974
+
2975
+ std::stringstream cf_ids_oss;
2976
+ bool empty_cfs = true;
2977
+ for (uint32_t cf_id : encountered_cf_ids) {
2978
+ if (ucmps.find(cf_id) == ucmps.end()) {
2979
+ if (empty_cfs) {
2980
+ cf_ids_oss << "[";
2981
+ empty_cfs = false;
2982
+ } else {
2983
+ cf_ids_oss << ",";
2984
+ }
2985
+ cf_ids_oss << cf_id;
2986
+ }
2987
+ }
2988
+ if (!empty_cfs) {
2989
+ cf_ids_oss << "]";
2990
+ std::cout
2991
+ << "(Column family id: " << cf_ids_oss.str()
2992
+ << " contained in WAL are not opened in DB. Applied default "
2993
+ "hex formatting for user key. Specify --db=<db_path> to "
2994
+ "open DB for better user key formatting if it contains timestamp.)"
2995
+ << std::endl;
2996
+ }
2787
2997
  }
2788
2998
  }
2789
2999
 
@@ -2799,7 +3009,7 @@ WALDumperCommand::WALDumperCommand(
2799
3009
  const std::map<std::string, std::string>& options,
2800
3010
  const std::vector<std::string>& flags)
2801
3011
  : LDBCommand(options, flags, true,
2802
- BuildCmdLineOptions({ARG_WAL_FILE, ARG_WRITE_COMMITTED,
3012
+ BuildCmdLineOptions({ARG_WAL_FILE, ARG_DB, ARG_WRITE_COMMITTED,
2803
3013
  ARG_PRINT_HEADER, ARG_PRINT_VALUE})),
2804
3014
  print_header_(false),
2805
3015
  print_values_(false),
@@ -2819,12 +3029,17 @@ WALDumperCommand::WALDumperCommand(
2819
3029
  exec_state_ = LDBCommandExecuteResult::Failed("Argument " + ARG_WAL_FILE +
2820
3030
  " must be specified.");
2821
3031
  }
3032
+
3033
+ if (!db_path_.empty()) {
3034
+ no_db_open_ = false;
3035
+ }
2822
3036
  }
2823
3037
 
2824
3038
  void WALDumperCommand::Help(std::string& ret) {
2825
3039
  ret.append(" ");
2826
3040
  ret.append(WALDumperCommand::Name());
2827
3041
  ret.append(" --" + ARG_WAL_FILE + "=<write_ahead_log_file_path>");
3042
+ ret.append(" [--" + ARG_DB + "=<db_path>]");
2828
3043
  ret.append(" [--" + ARG_PRINT_HEADER + "] ");
2829
3044
  ret.append(" [--" + ARG_PRINT_VALUE + "] ");
2830
3045
  ret.append(" [--" + ARG_WRITE_COMMITTED + "=true|false] ");
@@ -2834,7 +3049,7 @@ void WALDumperCommand::Help(std::string& ret) {
2834
3049
  void WALDumperCommand::DoCommand() {
2835
3050
  PrepareOptions();
2836
3051
  DumpWalFile(options_, wal_file_, print_header_, print_values_,
2837
- is_write_committed_, &exec_state_);
3052
+ is_write_committed_, ucmps_, &exec_state_);
2838
3053
  }
2839
3054
 
2840
3055
  // ----------------------------------------------------------------------------
@@ -2842,9 +3057,9 @@ void WALDumperCommand::DoCommand() {
2842
3057
  GetCommand::GetCommand(const std::vector<std::string>& params,
2843
3058
  const std::map<std::string, std::string>& options,
2844
3059
  const std::vector<std::string>& flags)
2845
- : LDBCommand(
2846
- options, flags, true,
2847
- BuildCmdLineOptions({ARG_TTL, ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX})) {
3060
+ : LDBCommand(options, flags, true,
3061
+ BuildCmdLineOptions({ARG_TTL, ARG_HEX, ARG_KEY_HEX,
3062
+ ARG_VALUE_HEX, ARG_READ_TIMESTAMP})) {
2848
3063
  if (params.size() != 1) {
2849
3064
  exec_state_ = LDBCommandExecuteResult::Failed(
2850
3065
  "<key> must be specified for the get command");
@@ -2861,6 +3076,7 @@ void GetCommand::Help(std::string& ret) {
2861
3076
  ret.append(" ");
2862
3077
  ret.append(GetCommand::Name());
2863
3078
  ret.append(" <key>");
3079
+ ret.append(" [--" + ARG_READ_TIMESTAMP + "=<uint64_ts>] ");
2864
3080
  ret.append(" [--" + ARG_TTL + "]");
2865
3081
  ret.append("\n");
2866
3082
  }
@@ -2870,8 +3086,18 @@ void GetCommand::DoCommand() {
2870
3086
  assert(GetExecuteState().IsFailed());
2871
3087
  return;
2872
3088
  }
3089
+ ReadOptions ropts;
3090
+ Slice read_timestamp;
3091
+ ColumnFamilyHandle* cfh = GetCfHandle();
3092
+ Status st = MaybePopulateReadTimestamp(cfh, ropts, &read_timestamp);
3093
+ if (!st.ok()) {
3094
+ std::stringstream oss;
3095
+ oss << "Get failed: " << st.ToString();
3096
+ exec_state_ = LDBCommandExecuteResult::Failed(oss.str());
3097
+ return;
3098
+ }
2873
3099
  std::string value;
2874
- Status st = db_->Get(ReadOptions(), GetCfHandle(), key_, &value);
3100
+ st = db_->Get(ropts, cfh, key_, &value);
2875
3101
  if (st.ok()) {
2876
3102
  fprintf(stdout, "%s\n",
2877
3103
  (is_value_hex_ ? StringToHex(value) : value).c_str());
@@ -2891,7 +3117,8 @@ MultiGetCommand::MultiGetCommand(
2891
3117
  const std::map<std::string, std::string>& options,
2892
3118
  const std::vector<std::string>& flags)
2893
3119
  : LDBCommand(options, flags, true,
2894
- BuildCmdLineOptions({ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX})) {
3120
+ BuildCmdLineOptions({ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX,
3121
+ ARG_READ_TIMESTAMP})) {
2895
3122
  if (params.size() < 1) {
2896
3123
  exec_state_ = LDBCommandExecuteResult::Failed(
2897
3124
  "At least one <key> must be specified for multi_get.");
@@ -2907,6 +3134,7 @@ void MultiGetCommand::Help(std::string& ret) {
2907
3134
  ret.append(" ");
2908
3135
  ret.append(MultiGetCommand::Name());
2909
3136
  ret.append(" <key_1> <key_2> <key_3> ...");
3137
+ ret.append(" [--" + ARG_READ_TIMESTAMP + "=<uint64_ts>] ");
2910
3138
  ret.append("\n");
2911
3139
  }
2912
3140
 
@@ -2915,6 +3143,16 @@ void MultiGetCommand::DoCommand() {
2915
3143
  assert(GetExecuteState().IsFailed());
2916
3144
  return;
2917
3145
  }
3146
+ ReadOptions ropts;
3147
+ Slice read_timestamp;
3148
+ ColumnFamilyHandle* cfh = GetCfHandle();
3149
+ Status st = MaybePopulateReadTimestamp(cfh, ropts, &read_timestamp);
3150
+ if (!st.ok()) {
3151
+ std::stringstream oss;
3152
+ oss << "MultiGet failed: " << st.ToString();
3153
+ exec_state_ = LDBCommandExecuteResult::Failed(oss.str());
3154
+ return;
3155
+ }
2918
3156
  size_t num_keys = keys_.size();
2919
3157
  std::vector<Slice> key_slices;
2920
3158
  std::vector<PinnableSlice> values(num_keys);
@@ -2922,8 +3160,8 @@ void MultiGetCommand::DoCommand() {
2922
3160
  for (const std::string& key : keys_) {
2923
3161
  key_slices.emplace_back(key);
2924
3162
  }
2925
- db_->MultiGet(ReadOptions(), GetCfHandle(), num_keys, key_slices.data(),
2926
- values.data(), statuses.data());
3163
+ db_->MultiGet(ropts, cfh, num_keys, key_slices.data(), values.data(),
3164
+ statuses.data());
2927
3165
 
2928
3166
  bool failed = false;
2929
3167
  for (size_t i = 0; i < num_keys; ++i) {
@@ -2938,7 +3176,7 @@ void MultiGetCommand::DoCommand() {
2938
3176
  fprintf(stderr, "Status for key %s: %s\n",
2939
3177
  (is_key_hex_ ? StringToHex(keys_[i]) : keys_[i]).c_str(),
2940
3178
  statuses[i].ToString().c_str());
2941
- failed = false;
3179
+ failed = true;
2942
3180
  }
2943
3181
  }
2944
3182
  if (failed) {
@@ -2953,9 +3191,9 @@ GetEntityCommand::GetEntityCommand(
2953
3191
  const std::vector<std::string>& params,
2954
3192
  const std::map<std::string, std::string>& options,
2955
3193
  const std::vector<std::string>& flags)
2956
- : LDBCommand(
2957
- options, flags, true,
2958
- BuildCmdLineOptions({ARG_TTL, ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX})) {
3194
+ : LDBCommand(options, flags, true,
3195
+ BuildCmdLineOptions({ARG_TTL, ARG_HEX, ARG_KEY_HEX,
3196
+ ARG_VALUE_HEX, ARG_READ_TIMESTAMP})) {
2959
3197
  if (params.size() != 1) {
2960
3198
  exec_state_ = LDBCommandExecuteResult::Failed(
2961
3199
  "<key> must be specified for the get_entity command");
@@ -2972,6 +3210,7 @@ void GetEntityCommand::Help(std::string& ret) {
2972
3210
  ret.append(" ");
2973
3211
  ret.append(GetEntityCommand::Name());
2974
3212
  ret.append(" <key>");
3213
+ ret.append(" [--" + ARG_READ_TIMESTAMP + "=<uint64_ts>] ");
2975
3214
  ret.append(" [--" + ARG_TTL + "]");
2976
3215
  ret.append("\n");
2977
3216
  }
@@ -2981,9 +3220,18 @@ void GetEntityCommand::DoCommand() {
2981
3220
  assert(GetExecuteState().IsFailed());
2982
3221
  return;
2983
3222
  }
3223
+ ReadOptions ropt;
3224
+ Slice read_timestamp;
3225
+ ColumnFamilyHandle* cfh = GetCfHandle();
3226
+ Status st = MaybePopulateReadTimestamp(cfh, ropt, &read_timestamp);
3227
+ if (!st.ok()) {
3228
+ std::stringstream oss;
3229
+ oss << "GetEntity failed: " << st.ToString();
3230
+ exec_state_ = LDBCommandExecuteResult::Failed(oss.str());
3231
+ return;
3232
+ }
2984
3233
  PinnableWideColumns pinnable_wide_columns;
2985
- Status st = db_->GetEntity(ReadOptions(), GetCfHandle(), key_,
2986
- &pinnable_wide_columns);
3234
+ st = db_->GetEntity(ropt, cfh, key_, &pinnable_wide_columns);
2987
3235
  if (st.ok()) {
2988
3236
  std::ostringstream oss;
2989
3237
  WideColumnsHelper::DumpWideColumns(pinnable_wide_columns.columns(), oss,
@@ -2998,6 +3246,85 @@ void GetEntityCommand::DoCommand() {
2998
3246
 
2999
3247
  // ----------------------------------------------------------------------------
3000
3248
 
3249
+ MultiGetEntityCommand::MultiGetEntityCommand(
3250
+ const std::vector<std::string>& params,
3251
+ const std::map<std::string, std::string>& options,
3252
+ const std::vector<std::string>& flags)
3253
+ : LDBCommand(options, flags, true /* is_read_only */,
3254
+ BuildCmdLineOptions({ARG_HEX, ARG_KEY_HEX, ARG_VALUE_HEX,
3255
+ ARG_READ_TIMESTAMP})) {
3256
+ if (params.size() < 1) {
3257
+ exec_state_ = LDBCommandExecuteResult::Failed(
3258
+ "At least one <key> must be specified for the multi_get_entity "
3259
+ "command");
3260
+ } else {
3261
+ for (size_t i = 0; i < params.size(); i++) {
3262
+ std::string key = params.at(i);
3263
+ keys_.emplace_back(is_key_hex_ ? HexToString(key) : key);
3264
+ }
3265
+ }
3266
+ }
3267
+
3268
+ void MultiGetEntityCommand::Help(std::string& ret) {
3269
+ ret.append(" ");
3270
+ ret.append(MultiGetEntityCommand::Name());
3271
+ ret.append(" <key_1> <key_2> <key_3> ...");
3272
+ ret.append(" [--" + ARG_READ_TIMESTAMP + "=<uint64_ts>] ");
3273
+ ret.append("\n");
3274
+ }
3275
+
3276
+ void MultiGetEntityCommand::DoCommand() {
3277
+ if (!db_) {
3278
+ assert(GetExecuteState().IsFailed());
3279
+ return;
3280
+ }
3281
+
3282
+ ReadOptions ropt;
3283
+ Slice read_timestamp;
3284
+ ColumnFamilyHandle* cfh = GetCfHandle();
3285
+ Status st = MaybePopulateReadTimestamp(cfh, ropt, &read_timestamp);
3286
+ if (!st.ok()) {
3287
+ std::stringstream oss;
3288
+ oss << "MultiGetEntity failed: " << st.ToString();
3289
+ exec_state_ = LDBCommandExecuteResult::Failed(oss.str());
3290
+ return;
3291
+ }
3292
+ size_t num_keys = keys_.size();
3293
+ std::vector<Slice> key_slices;
3294
+ std::vector<PinnableWideColumns> results(num_keys);
3295
+ std::vector<Status> statuses(num_keys);
3296
+ for (const std::string& key : keys_) {
3297
+ key_slices.emplace_back(key);
3298
+ }
3299
+
3300
+ db_->MultiGetEntity(ropt, cfh, num_keys, key_slices.data(), results.data(),
3301
+ statuses.data());
3302
+
3303
+ bool failed = false;
3304
+ for (size_t i = 0; i < num_keys; ++i) {
3305
+ std::string key = is_key_hex_ ? StringToHex(keys_[i]) : keys_[i];
3306
+ if (statuses[i].ok()) {
3307
+ std::ostringstream oss;
3308
+ oss << key << DELIM;
3309
+ WideColumnsHelper::DumpWideColumns(results[i].columns(), oss,
3310
+ is_value_hex_);
3311
+ fprintf(stdout, "%s\n", oss.str().c_str());
3312
+ } else if (statuses[i].IsNotFound()) {
3313
+ fprintf(stdout, "Key not found: %s\n", key.c_str());
3314
+ } else {
3315
+ fprintf(stderr, "Status for key %s: %s\n", key.c_str(),
3316
+ statuses[i].ToString().c_str());
3317
+ failed = true;
3318
+ }
3319
+ }
3320
+ if (failed) {
3321
+ exec_state_ =
3322
+ LDBCommandExecuteResult::Failed("one or more keys had non-okay status");
3323
+ }
3324
+ }
3325
+
3326
+ // ----------------------------------------------------------------------------
3327
+
3001
3328
  ApproxSizeCommand::ApproxSizeCommand(
3002
3329
  const std::vector<std::string>& /*params*/,
3003
3330
  const std::map<std::string, std::string>& options,
@@ -3129,11 +3456,11 @@ void BatchPutCommand::OverrideBaseOptions() {
3129
3456
  ScanCommand::ScanCommand(const std::vector<std::string>& /*params*/,
3130
3457
  const std::map<std::string, std::string>& options,
3131
3458
  const std::vector<std::string>& flags)
3132
- : LDBCommand(
3133
- options, flags, true,
3134
- BuildCmdLineOptions({ARG_TTL, ARG_NO_VALUE, ARG_HEX, ARG_KEY_HEX,
3135
- ARG_TO, ARG_VALUE_HEX, ARG_FROM, ARG_TIMESTAMP,
3136
- ARG_MAX_KEYS, ARG_TTL_START, ARG_TTL_END})),
3459
+ : LDBCommand(options, flags, true,
3460
+ BuildCmdLineOptions(
3461
+ {ARG_TTL, ARG_NO_VALUE, ARG_HEX, ARG_KEY_HEX, ARG_TO,
3462
+ ARG_VALUE_HEX, ARG_FROM, ARG_TIMESTAMP, ARG_MAX_KEYS,
3463
+ ARG_TTL_START, ARG_TTL_END, ARG_READ_TIMESTAMP})),
3137
3464
  start_key_specified_(false),
3138
3465
  end_key_specified_(false),
3139
3466
  max_keys_scanned_(-1),
@@ -3189,6 +3516,7 @@ void ScanCommand::Help(std::string& ret) {
3189
3516
  ret.append(" [--" + ARG_TTL_START + "=<N>:- is inclusive]");
3190
3517
  ret.append(" [--" + ARG_TTL_END + "=<N>:- is exclusive]");
3191
3518
  ret.append(" [--" + ARG_NO_VALUE + "]");
3519
+ ret.append(" [--" + ARG_READ_TIMESTAMP + "=<uint64_ts>] ");
3192
3520
  ret.append("\n");
3193
3521
  }
3194
3522
 
@@ -3200,8 +3528,19 @@ void ScanCommand::DoCommand() {
3200
3528
 
3201
3529
  int num_keys_scanned = 0;
3202
3530
  ReadOptions scan_read_opts;
3531
+ ColumnFamilyHandle* cfh = GetCfHandle();
3532
+ const Comparator* ucmp = cfh->GetComparator();
3533
+ size_t ts_sz = ucmp->timestamp_size();
3534
+ Slice read_timestamp;
3535
+ Status st = MaybePopulateReadTimestamp(cfh, scan_read_opts, &read_timestamp);
3536
+ if (!st.ok()) {
3537
+ std::stringstream oss;
3538
+ oss << "Scan failed: " << st.ToString();
3539
+ exec_state_ = LDBCommandExecuteResult::Failed(oss.str());
3540
+ return;
3541
+ }
3203
3542
  scan_read_opts.total_order_seek = true;
3204
- Iterator* it = db_->NewIterator(scan_read_opts, GetCfHandle());
3543
+ Iterator* it = db_->NewIterator(scan_read_opts, cfh);
3205
3544
  if (start_key_specified_) {
3206
3545
  it->Seek(start_key_);
3207
3546
  } else {
@@ -3248,12 +3587,15 @@ void ScanCommand::DoCommand() {
3248
3587
  }
3249
3588
  fprintf(stdout, "%s\n", key_str.c_str());
3250
3589
  } else {
3251
- std::string str = is_db_ttl_ ? PrintKeyValue(it->key().ToString(),
3252
- it->value().ToString(),
3253
- is_key_hex_, is_value_hex_)
3254
- : PrintKeyValueOrWideColumns(
3255
- it->key(), it->value(), it->columns(),
3256
- is_key_hex_, is_value_hex_);
3590
+ std::string str =
3591
+ is_db_ttl_
3592
+ ? PrintKeyValue(it->key().ToString(),
3593
+ ts_sz == 0 ? "" : it->timestamp().ToString(),
3594
+ it->value().ToString(), is_key_hex_,
3595
+ is_value_hex_, ucmp)
3596
+ : PrintKeyValueOrWideColumns(
3597
+ it->key(), ts_sz == 0 ? "" : it->timestamp(), it->value(),
3598
+ it->columns(), is_key_hex_, is_value_hex_, ucmp);
3257
3599
  fprintf(stdout, "%s\n", str.c_str());
3258
3600
  }
3259
3601
 
@@ -3473,7 +3815,7 @@ PutEntityCommand::PutEntityCommand(
3473
3815
 
3474
3816
  void PutEntityCommand::Help(std::string& ret) {
3475
3817
  ret.append(" ");
3476
- ret.append(PutCommand::Name());
3818
+ ret.append(PutEntityCommand::Name());
3477
3819
  ret.append(
3478
3820
  " <key> <column1_name>:<column1_value> <column2_name>:<column2_value> "
3479
3821
  "<...>");
@@ -3512,6 +3854,7 @@ const char* DBQuerierCommand::HELP_CMD = "help";
3512
3854
  const char* DBQuerierCommand::GET_CMD = "get";
3513
3855
  const char* DBQuerierCommand::PUT_CMD = "put";
3514
3856
  const char* DBQuerierCommand::DELETE_CMD = "delete";
3857
+ const char* DBQuerierCommand::COUNT_CMD = "count";
3515
3858
 
3516
3859
  DBQuerierCommand::DBQuerierCommand(
3517
3860
  const std::vector<std::string>& /*params*/,
@@ -3540,72 +3883,211 @@ void DBQuerierCommand::DoCommand() {
3540
3883
  return;
3541
3884
  }
3542
3885
 
3543
- ReadOptions read_options;
3544
- WriteOptions write_options;
3545
-
3546
3886
  std::string line;
3547
- std::string key;
3548
- std::string value;
3549
3887
  Status s;
3550
- std::stringstream oss;
3551
- while (s.ok() && getline(std::cin, line, '\n')) {
3888
+ ColumnFamilyHandle* cfh = GetCfHandle();
3889
+ const Comparator* ucmp = cfh->GetComparator();
3890
+ while ((s.ok() || s.IsNotFound() || s.IsInvalidArgument()) &&
3891
+ getline(std::cin, line, '\n')) {
3892
+ std::string key;
3893
+ std::string timestamp;
3894
+ std::string value;
3895
+ // Reset to OK status before parsing and executing next user command.
3896
+ s = Status::OK();
3897
+ std::stringstream oss;
3552
3898
  // Parse line into std::vector<std::string>
3553
3899
  std::vector<std::string> tokens;
3900
+ ParsedParams parsed_params;
3554
3901
  size_t pos = 0;
3555
3902
  while (true) {
3556
3903
  size_t pos2 = line.find(' ', pos);
3904
+ std::string token =
3905
+ line.substr(pos, (pos2 == std::string::npos) ? pos2 : (pos2 - pos));
3906
+ ParseSingleParam(token, parsed_params, tokens);
3557
3907
  if (pos2 == std::string::npos) {
3558
3908
  break;
3559
3909
  }
3560
- tokens.push_back(line.substr(pos, pos2 - pos));
3561
3910
  pos = pos2 + 1;
3562
3911
  }
3563
- tokens.push_back(line.substr(pos));
3912
+
3913
+ if (tokens.empty() || !parsed_params.flags.empty()) {
3914
+ fprintf(stdout, "Bad command\n");
3915
+ continue;
3916
+ }
3564
3917
 
3565
3918
  const std::string& cmd = tokens[0];
3919
+ ReadOptions read_options;
3920
+ WriteOptions write_options;
3921
+ Slice read_timestamp;
3566
3922
 
3567
3923
  if (cmd == HELP_CMD) {
3568
3924
  fprintf(stdout,
3569
- "get <key>\n"
3570
- "put <key> <value>\n"
3571
- "delete <key>\n");
3572
- } else if (cmd == DELETE_CMD && tokens.size() == 2) {
3925
+ "get <key> [--read_timestamp=<uint64_ts>]\n"
3926
+ "put <key> [<write_timestamp>] <value>\n"
3927
+ "delete <key> [<write_timestamp>]\n"
3928
+ "count [--from=<start_key>] [--to=<end_key>] "
3929
+ "[--read_timestamp=<uint64_ts>]\n");
3930
+ } else if (cmd == DELETE_CMD && parsed_params.option_map.empty()) {
3573
3931
  key = (is_key_hex_ ? HexToString(tokens[1]) : tokens[1]);
3574
- s = db_->Delete(write_options, GetCfHandle(), Slice(key));
3932
+ if (tokens.size() == 2) {
3933
+ s = db_->Delete(write_options, cfh, Slice(key));
3934
+ } else if (tokens.size() == 3) {
3935
+ Status encode_s = EncodeUserProvidedTimestamp(tokens[2], &timestamp);
3936
+ if (encode_s.ok()) {
3937
+ s = db_->Delete(write_options, cfh, Slice(key), Slice(timestamp));
3938
+ } else {
3939
+ fprintf(stdout, "delete gets invalid argument: %s\n",
3940
+ encode_s.ToString().c_str());
3941
+ continue;
3942
+ }
3943
+ } else {
3944
+ fprintf(stdout, "delete gets invalid arguments\n");
3945
+ continue;
3946
+ }
3947
+ oss << "delete " << (is_key_hex_ ? StringToHex(key) : key);
3948
+ if (!timestamp.empty()) {
3949
+ oss << " write_ts: " << ucmp->TimestampToString(timestamp);
3950
+ }
3575
3951
  if (s.ok()) {
3576
- fprintf(stdout, "Successfully deleted %s\n", tokens[1].c_str());
3952
+ oss << " succeeded";
3577
3953
  } else {
3578
- oss << "delete " << key << " failed: " << s.ToString();
3954
+ oss << " failed: " << s.ToString();
3579
3955
  }
3580
- } else if (cmd == PUT_CMD && tokens.size() == 3) {
3956
+ fprintf(stdout, "%s\n", oss.str().c_str());
3957
+ } else if (cmd == PUT_CMD && parsed_params.option_map.empty()) {
3581
3958
  key = (is_key_hex_ ? HexToString(tokens[1]) : tokens[1]);
3582
- value = (is_value_hex_ ? HexToString(tokens[2]) : tokens[2]);
3583
- s = db_->Put(write_options, GetCfHandle(), Slice(key), Slice(value));
3959
+ if (tokens.size() == 3) {
3960
+ value = (is_value_hex_ ? HexToString(tokens[2]) : tokens[2]);
3961
+ s = db_->Put(write_options, cfh, Slice(key), Slice(value));
3962
+ } else if (tokens.size() == 4) {
3963
+ value = (is_value_hex_ ? HexToString(tokens[3]) : tokens[3]);
3964
+ Status encode_s = EncodeUserProvidedTimestamp(tokens[2], &timestamp);
3965
+ if (encode_s.ok()) {
3966
+ s = db_->Put(write_options, cfh, Slice(key), Slice(timestamp),
3967
+ Slice(value));
3968
+ } else {
3969
+ fprintf(stdout, "put gets invalid argument: %s\n",
3970
+ encode_s.ToString().c_str());
3971
+ continue;
3972
+ }
3973
+ } else {
3974
+ fprintf(stdout, "put gets invalid arguments\n");
3975
+ continue;
3976
+ }
3977
+
3978
+ oss << "put " << (is_key_hex_ ? StringToHex(key) : key);
3979
+ if (!timestamp.empty()) {
3980
+ oss << " write_ts: " << ucmp->TimestampToString(timestamp);
3981
+ }
3982
+ oss << " => " << (is_value_hex_ ? StringToHex(value) : value);
3584
3983
  if (s.ok()) {
3585
- fprintf(stdout, "Successfully put %s %s\n", tokens[1].c_str(),
3586
- tokens[2].c_str());
3984
+ oss << " succeeded";
3587
3985
  } else {
3588
- oss << "put " << key << "=>" << value << " failed: " << s.ToString();
3986
+ oss << " failed: " << s.ToString();
3589
3987
  }
3988
+ fprintf(stdout, "%s\n", oss.str().c_str());
3590
3989
  } else if (cmd == GET_CMD && tokens.size() == 2) {
3591
3990
  key = (is_key_hex_ ? HexToString(tokens[1]) : tokens[1]);
3592
- s = db_->Get(read_options, GetCfHandle(), Slice(key), &value);
3991
+ bool bad_option = false;
3992
+ for (auto& option : parsed_params.option_map) {
3993
+ if (option.first == "read_timestamp") {
3994
+ Status encode_s =
3995
+ EncodeUserProvidedTimestamp(option.second, &timestamp);
3996
+ if (!encode_s.ok()) {
3997
+ fprintf(stdout, "get gets invalid argument: %s\n",
3998
+ encode_s.ToString().c_str());
3999
+ bad_option = true;
4000
+ break;
4001
+ }
4002
+ read_timestamp = timestamp;
4003
+ read_options.timestamp = &read_timestamp;
4004
+ } else {
4005
+ fprintf(stdout, "get gets invalid arguments\n");
4006
+ bad_option = true;
4007
+ break;
4008
+ }
4009
+ }
4010
+ if (bad_option) {
4011
+ continue;
4012
+ }
4013
+ s = db_->Get(read_options, cfh, Slice(key), &value);
3593
4014
  if (s.ok()) {
3594
4015
  fprintf(stdout, "%s\n",
3595
- PrintKeyValue(key, value, is_key_hex_, is_value_hex_).c_str());
4016
+ PrintKeyValue(key, timestamp, value, is_key_hex_, is_value_hex_,
4017
+ ucmp)
4018
+ .c_str());
3596
4019
  } else {
3597
- if (s.IsNotFound()) {
3598
- fprintf(stdout, "Not found %s\n", tokens[1].c_str());
4020
+ oss << "get " << (is_key_hex_ ? StringToHex(key) : key);
4021
+ if (!timestamp.empty()) {
4022
+ oss << " read_timestamp: " << ucmp->TimestampToString(timestamp);
4023
+ }
4024
+ oss << " status: " << s.ToString();
4025
+ fprintf(stdout, "%s\n", oss.str().c_str());
4026
+ }
4027
+ } else if (cmd == COUNT_CMD) {
4028
+ std::string start_key;
4029
+ std::string end_key;
4030
+ bool bad_option = false;
4031
+ for (auto& option : parsed_params.option_map) {
4032
+ if (option.first == "from") {
4033
+ start_key =
4034
+ (is_key_hex_ ? HexToString(option.second) : option.second);
4035
+ } else if (option.first == "to") {
4036
+ end_key = (is_key_hex_ ? HexToString(option.second) : option.second);
4037
+ } else if (option.first == "read_timestamp") {
4038
+ Status encode_s =
4039
+ EncodeUserProvidedTimestamp(option.second, &timestamp);
4040
+ if (!encode_s.ok()) {
4041
+ bad_option = true;
4042
+ fprintf(stdout, "count gets invalid argument: %s\n",
4043
+ encode_s.ToString().c_str());
4044
+ break;
4045
+ }
4046
+ read_timestamp = timestamp;
4047
+ read_options.timestamp = &read_timestamp;
3599
4048
  } else {
3600
- oss << "get " << key << " error: " << s.ToString();
4049
+ fprintf(stdout, "count gets invalid arguments\n");
4050
+ bad_option = true;
4051
+ break;
4052
+ }
4053
+ }
4054
+ if (bad_option) {
4055
+ continue;
4056
+ }
4057
+
4058
+ Slice end_key_slice(end_key);
4059
+ uint64_t count = 0;
4060
+ if (!end_key.empty()) {
4061
+ read_options.iterate_upper_bound = &end_key_slice;
4062
+ }
4063
+ std::unique_ptr<Iterator> iter(db_->NewIterator(read_options, cfh));
4064
+ if (start_key.empty()) {
4065
+ iter->SeekToFirst();
4066
+ } else {
4067
+ iter->Seek(start_key);
4068
+ }
4069
+ while (iter->status().ok() && iter->Valid()) {
4070
+ count++;
4071
+ iter->Next();
4072
+ }
4073
+ if (iter->status().ok()) {
4074
+ fprintf(stdout, "%" PRIu64 "\n", count);
4075
+ } else {
4076
+ oss << "scan from "
4077
+ << (is_key_hex_ ? StringToHex(start_key) : start_key);
4078
+ if (!timestamp.empty()) {
4079
+ oss << " read_timestamp: " << ucmp->TimestampToString(timestamp);
3601
4080
  }
4081
+ oss << " to " << (is_key_hex_ ? StringToHex(end_key) : end_key)
4082
+ << " failed: " << iter->status().ToString();
4083
+ fprintf(stdout, "%s\n", oss.str().c_str());
3602
4084
  }
3603
4085
  } else {
3604
4086
  fprintf(stdout, "Unknown command %s\n", line.c_str());
3605
4087
  }
3606
4088
  }
3607
- if (!s.ok()) {
3608
- exec_state_ = LDBCommandExecuteResult::Failed(oss.str());
4089
+ if (!(s.ok() || s.IsNotFound() || s.IsInvalidArgument())) {
4090
+ exec_state_ = LDBCommandExecuteResult::Failed(s.ToString());
3609
4091
  }
3610
4092
  }
3611
4093
 
@@ -3941,6 +4423,18 @@ void DumpBlobFile(const std::string& filename, bool is_key_hex,
3941
4423
  fprintf(stderr, "Failed: %s\n", s.ToString().c_str());
3942
4424
  }
3943
4425
  }
4426
+
4427
+ Status EncodeUserProvidedTimestamp(const std::string& user_timestamp,
4428
+ std::string* ts_buf) {
4429
+ uint64_t int_timestamp;
4430
+ std::istringstream iss(user_timestamp);
4431
+ if (!(iss >> int_timestamp)) {
4432
+ return Status::InvalidArgument(
4433
+ "user provided timestamp is not a valid uint64 value.");
4434
+ }
4435
+ EncodeU64Ts(int_timestamp, ts_buf);
4436
+ return Status::OK();
4437
+ }
3944
4438
  } // namespace
3945
4439
 
3946
4440
  DBFileDumperCommand::DBFileDumperCommand(
@@ -4035,7 +4529,7 @@ void DBFileDumperCommand::DoCommand() {
4035
4529
 
4036
4530
  std::cout << "Write Ahead Log Files" << std::endl;
4037
4531
  std::cout << "==============================" << std::endl;
4038
- ROCKSDB_NAMESPACE::VectorLogPtr wal_files;
4532
+ ROCKSDB_NAMESPACE::VectorWalPtr wal_files;
4039
4533
  s = db_->GetSortedWalFiles(wal_files);
4040
4534
  if (!s.ok()) {
4041
4535
  std::cerr << "Error when getting WAL files" << std::endl;
@@ -4052,7 +4546,7 @@ void DBFileDumperCommand::DoCommand() {
4052
4546
  std::cout << filename << std::endl;
4053
4547
  // TODO(myabandeh): allow configuring is_write_commited
4054
4548
  DumpWalFile(options_, filename, true, true, true /* is_write_commited */,
4055
- &exec_state_);
4549
+ ucmps_, &exec_state_);
4056
4550
  }
4057
4551
  }
4058
4552
  }