ladybug 0.20.2__tar.gz → 0.20.2.dev20260908__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (114) hide show
  1. sdist/PKG-INFO +1 -1
  2. sdist/ladybug-source/CMakeLists.txt +2 -2
  3. sdist/ladybug-source/docs/multi_parent_lifetime.md +12 -0
  4. sdist/ladybug-source/scripts/antlr4/Cypher.g4 +1 -1
  5. sdist/ladybug-source/scripts/antlr4/hash.md5 +1 -1
  6. sdist/ladybug-source/scripts/pip-package/setup.py +13 -2
  7. sdist/ladybug-source/scripts/pip-package/split_debug_symbols.py +262 -0
  8. sdist/ladybug-source/src/antlr4/Cypher.g4 +1 -1
  9. sdist/ladybug-source/src/binder/bind/bind_export_database.cpp +107 -27
  10. sdist/ladybug-source/src/binder/bind/bind_import_database.cpp +100 -1
  11. sdist/ladybug-source/src/binder/query/query_graph.cpp +15 -8
  12. sdist/ladybug-source/src/catalog/catalog_entry/node_table_catalog_entry.cpp +3 -0
  13. sdist/ladybug-source/src/catalog/catalog_entry/rel_group_catalog_entry.cpp +5 -1
  14. sdist/ladybug-source/src/function/export/CMakeLists.txt +2 -1
  15. sdist/ladybug-source/src/function/export/export_indptr_function.cpp +171 -0
  16. sdist/ladybug-source/src/function/export/export_parquet_function.cpp +18 -2
  17. sdist/ladybug-source/src/include/binder/bound_export_database.h +3 -0
  18. sdist/ladybug-source/src/include/common/constants.h +5 -0
  19. sdist/ladybug-source/src/include/common/data_chunk/data_chunk_state.h +34 -54
  20. sdist/ladybug-source/src/include/common/types/types.h +5 -1
  21. sdist/ladybug-source/src/include/function/export/export_function.h +8 -0
  22. sdist/ladybug-source/src/include/function/hash/hash_functions.h +6 -1
  23. sdist/ladybug-source/src/include/main/client_config.h +26 -0
  24. sdist/ladybug-source/src/include/main/client_context.h +3 -0
  25. sdist/ladybug-source/src/include/main/settings.h +10 -0
  26. sdist/ladybug-source/src/include/optimizer/count_rel_table_optimizer.h +12 -0
  27. sdist/ladybug-source/src/include/optimizer/group_key_predicate_push_down_optimizer.h +49 -0
  28. sdist/ladybug-source/src/include/parser/expression/parsed_subquery_expression.h +1 -0
  29. sdist/ladybug-source/src/include/parser/port_db.h +3 -0
  30. sdist/ladybug-source/src/include/planner/operator/logical_operator.h +2 -0
  31. sdist/ladybug-source/src/include/planner/operator/scan/logical_count_anti_edge_chain.h +116 -0
  32. sdist/ladybug-source/src/include/planner/operator/scan/logical_count_extend_chain.h +108 -0
  33. sdist/ladybug-source/src/include/processor/operator/aggregate/base_aggregate.h +2 -0
  34. sdist/ladybug-source/src/include/processor/operator/aggregate/hash_aggregate.h +6 -0
  35. sdist/ladybug-source/src/include/processor/operator/base_partitioner_shared_state.h +9 -2
  36. sdist/ladybug-source/src/include/processor/operator/persistent/writer/parquet/parquet_writer.h +3 -1
  37. sdist/ladybug-source/src/include/processor/operator/physical_operator.h +2 -0
  38. sdist/ladybug-source/src/include/processor/operator/scan/count_anti_edge_chain.h +114 -0
  39. sdist/ladybug-source/src/include/processor/operator/scan/count_extend_chain.h +109 -0
  40. sdist/ladybug-source/src/include/processor/operator/scan/scan_node_table.h +16 -1
  41. sdist/ladybug-source/src/include/processor/operator/scan/scan_rel_table.h +21 -9
  42. sdist/ladybug-source/src/include/processor/plan_mapper.h +4 -0
  43. sdist/ladybug-source/src/include/processor/processor.h +38 -0
  44. sdist/ladybug-source/src/include/processor/result/result_set_descriptor.h +2 -2
  45. sdist/ladybug-source/src/include/storage/file_handle.h +3 -0
  46. sdist/ladybug-source/src/include/storage/index/index.h +10 -0
  47. sdist/ladybug-source/src/include/storage/storage_version_info.h +1 -1
  48. sdist/ladybug-source/src/include/storage/table/arrow_node_table.h +6 -1
  49. sdist/ladybug-source/src/include/storage/table/arrow_rel_table.h +4 -4
  50. sdist/ladybug-source/src/include/storage/table/arrow_table_support.h +18 -5
  51. sdist/ladybug-source/src/include/storage/table/csr_node_group.h +8 -0
  52. sdist/ladybug-source/src/include/storage/table/rel_table.h +33 -0
  53. sdist/ladybug-source/src/include/storage/table/table.h +6 -0
  54. sdist/ladybug-source/src/main/client_context.cpp +15 -0
  55. sdist/ladybug-source/src/main/db_config.cpp +2 -1
  56. sdist/ladybug-source/src/main/settings.cpp +46 -0
  57. sdist/ladybug-source/src/optimizer/CMakeLists.txt +1 -0
  58. sdist/ladybug-source/src/optimizer/count_rel_table_optimizer.cpp +675 -0
  59. sdist/ladybug-source/src/optimizer/filter_push_down_optimizer.cpp +6 -1
  60. sdist/ladybug-source/src/optimizer/group_key_predicate_push_down_optimizer.cpp +204 -0
  61. sdist/ladybug-source/src/optimizer/optimizer.cpp +162 -0
  62. sdist/ladybug-source/src/parser/expression/parsed_expression_visitor.cpp +23 -0
  63. sdist/ladybug-source/src/parser/transform/transform_port_db.cpp +5 -1
  64. sdist/ladybug-source/src/planner/operator/logical_operator.cpp +8 -2
  65. sdist/ladybug-source/src/planner/operator/scan/CMakeLists.txt +2 -0
  66. sdist/ladybug-source/src/planner/operator/scan/logical_count_anti_edge_chain.cpp +20 -0
  67. sdist/ladybug-source/src/planner/operator/scan/logical_count_extend_chain.cpp +21 -0
  68. sdist/ladybug-source/src/planner/plan/plan_join_order.cpp +54 -24
  69. sdist/ladybug-source/src/planner/plan/plan_port_db.cpp +6 -1
  70. sdist/ladybug-source/src/processor/map/CMakeLists.txt +2 -0
  71. sdist/ladybug-source/src/processor/map/map_aggregate.cpp +11 -1
  72. sdist/ladybug-source/src/processor/map/map_count_anti_edge_chain.cpp +52 -0
  73. sdist/ladybug-source/src/processor/map/map_count_extend_chain.cpp +52 -0
  74. sdist/ladybug-source/src/processor/map/map_merge.cpp +2 -2
  75. sdist/ladybug-source/src/processor/map/plan_mapper.cpp +6 -0
  76. sdist/ladybug-source/src/processor/operator/aggregate/hash_aggregate.cpp +10 -3
  77. sdist/ladybug-source/src/processor/operator/aggregate/packed_filtered_count.cpp +44 -6
  78. sdist/ladybug-source/src/processor/operator/base_partitioner_shared_state.cpp +5 -3
  79. sdist/ladybug-source/src/processor/operator/ddl/drop.cpp +4 -1
  80. sdist/ladybug-source/src/processor/operator/persistent/writer/parquet/parquet_writer.cpp +7 -2
  81. sdist/ladybug-source/src/processor/operator/physical_operator.cpp +4 -0
  82. sdist/ladybug-source/src/processor/operator/scan/CMakeLists.txt +2 -0
  83. sdist/ladybug-source/src/processor/operator/scan/count_anti_edge_chain.cpp +349 -0
  84. sdist/ladybug-source/src/processor/operator/scan/count_extend_chain.cpp +193 -0
  85. sdist/ladybug-source/src/processor/operator/scan/scan_node_table.cpp +71 -4
  86. sdist/ladybug-source/src/processor/operator/scan/scan_rel_table.cpp +22 -35
  87. sdist/ladybug-source/src/processor/operator/simple/export_db.cpp +30 -101
  88. sdist/ladybug-source/src/processor/processor.cpp +32 -0
  89. sdist/ladybug-source/src/processor/processor_task.cpp +14 -21
  90. sdist/ladybug-source/src/processor/result/factorized_table.cpp +7 -2
  91. sdist/ladybug-source/src/processor/result/pattern_creation_info_table.cpp +7 -2
  92. sdist/ladybug-source/src/processor/result/result_set_descriptor.cpp +2 -2
  93. sdist/ladybug-source/src/storage/file_handle.cpp +10 -7
  94. sdist/ladybug-source/src/storage/local_storage/local_node_table.cpp +12 -1
  95. sdist/ladybug-source/src/storage/shadow_file.cpp +43 -12
  96. sdist/ladybug-source/src/storage/storage_manager.cpp +12 -31
  97. sdist/ladybug-source/src/storage/table/arrow_node_table.cpp +13 -3
  98. sdist/ladybug-source/src/storage/table/arrow_rel_table.cpp +20 -7
  99. sdist/ladybug-source/src/storage/table/arrow_table_support.cpp +24 -20
  100. sdist/ladybug-source/src/storage/table/csr_node_group.cpp +93 -1
  101. sdist/ladybug-source/src/storage/table/ice_disk_node_table.cpp +26 -2
  102. sdist/ladybug-source/src/storage/table/node_group.cpp +18 -8
  103. sdist/ladybug-source/src/storage/table/node_table.cpp +129 -12
  104. sdist/ladybug-source/src/storage/table/rel_table.cpp +118 -15
  105. sdist/ladybug-source/src/storage/table/rel_table_data.cpp +17 -1
  106. sdist/ladybug-source/third_party/antlr4_cypher/cypher_parser.cpp +3323 -3272
  107. sdist/ladybug-source/third_party/antlr4_cypher/include/cypher_parser.h +1 -0
  108. sdist/ladybug-source/tools/python_api/CMakeLists.txt +81 -0
  109. sdist/ladybug-source/tools/python_api/src_cpp/include/pyarrow/pyarrow_scan.h +5 -0
  110. sdist/ladybug-source/tools/python_api/test/test_scan_pandas_pyarrow.py +12 -0
  111. sdist/ladybug.egg-info/PKG-INFO +1 -1
  112. sdist/pyproject.toml +1 -1
  113. sdist/setup.py +13 -2
  114. sdist/test/test_scan_pandas_pyarrow.py +12 -0
sdist/PKG-INFO CHANGED
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: ladybug
3
- Version: 0.20.2
3
+ Version: 0.20.2.dev20260908
4
4
  Summary: Highly scalable, extremely fast, easy-to-use embeddable graph database
5
5
  Home-page: https://github.com/LadybugDB/ladybug
6
6
  License: MIT
@@ -1,6 +1,6 @@
1
1
  cmake_minimum_required(VERSION 3.15)
2
2
 
3
- project(Lbug VERSION 0.20.2 LANGUAGES CXX C)
3
+ project(Lbug VERSION 0.20.2.20260908 LANGUAGES CXX C)
4
4
 
5
5
  option(SINGLE_THREADED "Single-threaded mode" FALSE)
6
6
  if(SINGLE_THREADED)
@@ -522,7 +522,7 @@ unset(LBUG_LINK_LIBRARIES)
522
522
 
523
523
  add_definitions(-DLBUG_ROOT_DIRECTORY="${PROJECT_SOURCE_DIR}")
524
524
  add_definitions(-DLBUG_CMAKE_VERSION="${CMAKE_PROJECT_VERSION}")
525
- add_definitions(-DLBUG_EXTENSION_VERSION="0.20.0")
525
+ add_definitions(-DLBUG_EXTENSION_VERSION="dev")
526
526
 
527
527
  if(BUILD_LBUG OR LBUG_API_USE_PRECOMPILED_LIB)
528
528
  include_directories(
@@ -1,5 +1,17 @@
1
1
  # Multi-parent packed child slices: lifetime & representation
2
2
 
3
+ > **Status update (multi-parent scan implemented):** the deferred change described below has
4
+ > been implemented. `PackedChildSlices` now holds a `shared_ptr<SelectionVector>` aliasing the
5
+ > bound (parent) chunk state's selection vector plus a prefix-sum `offsets` over ALL served
6
+ > parents (zeros allowed, zero-length ranges skipped by consumers). The CSR scan packs children
7
+ > of multiple parents per output batch in `CSRNodeGroupScanState::tryScanCachedTuplesPacked`
8
+ > (persistent with-cache path only). Multi-parent batches are enabled per-consumer:
9
+ > `ScanRelTable::setMultiParentPackedScanEnabled(true)` is set by the plan mapper only when the
10
+ > packed-aware `PackedFilteredCount` consumes the scan output directly. Plans that materialize
11
+ > the scan output across a `FactorizedTable` (e.g. when the filter predicate touches an nbr
12
+ > node property, which routes the property fetch through a hash join) keep the
13
+ > one-parent-per-batch contract — exactly the materialization hazard described below.
14
+
3
15
  Context: the `enable_packed_path_extend` path reuses the adjacency-list (CSR)
4
16
  rel scan to implement a physical packed extend. `ScanRelTable` attaches a
5
17
  `PackedChildSlices` descriptor to the **output** `DataChunkState`
@@ -326,7 +326,7 @@ iC_ExportDatabase
326
326
  : EXPORT SP DATABASE SP StringLiteral ( SP? '(' SP? iC_Options SP? ')' )? ;
327
327
 
328
328
  iC_ImportDatabase
329
- : IMPORT SP DATABASE SP StringLiteral;
329
+ : IMPORT SP DATABASE SP StringLiteral ( SP? '(' SP? iC_Options SP? ')' )? ;
330
330
 
331
331
  iC_AttachDatabase
332
332
  : ATTACH SP StringLiteral (SP AS SP oC_SchemaName)? SP '(' SP? DBTYPE SP oC_SymbolicName (SP? ',' SP? iC_Options)? SP? ')' ;
@@ -1 +1 @@
1
- c6ef6ea298ca84a5c080bae09a29c9db
1
+ 6fc9fc9085d4f1028a4fb2b319152cb5
@@ -91,16 +91,27 @@ class CMakeBuild(build_ext):
91
91
  if os.path.exists(flags_file):
92
92
  with open(flags_file) as f:
93
93
  extra_cmake_flags = f.read().strip()
94
+ # Wheel builds must not split symbols at build time: raw .debug/dSYM
95
+ # artifacts packaged inside the wheel break auditwheel/delocate
96
+ # repair. The split happens on the final repaired wheel instead (see
97
+ # split_debug_symbols.py), which needs the debug info to still be
98
+ # present in the shipped binary.
99
+ no_split_flag = '-DLBUG_SPLIT_DEBUG_SYMBOLS=OFF'
94
100
  if precompiled_lib_path:
95
101
  self.announce("Using precompiled liblbug from %s." % precompiled_lib_path)
96
102
  flags = (
97
- 'EXTRA_CMAKE_FLAGS=-DBUILD_LBUG=FALSE -DBUILD_SHELL=FALSE '
103
+ 'EXTRA_CMAKE_FLAGS=%s -DBUILD_LBUG=FALSE -DBUILD_SHELL=FALSE '
98
104
  '-DLBUG_API_USE_PRECOMPILED_LIB=TRUE '
99
- '-DLBUG_API_PRECOMPILED_LIB_PATH=%s' % precompiled_lib_path
105
+ '-DLBUG_API_PRECOMPILED_LIB_PATH=%s' % (no_split_flag, precompiled_lib_path)
100
106
  )
101
107
  if extra_cmake_flags:
102
108
  flags += ' ' + extra_cmake_flags
103
109
  full_cmd.append(flags)
110
+ else:
111
+ flags = 'EXTRA_CMAKE_FLAGS=%s' % no_split_flag
112
+ if extra_cmake_flags:
113
+ flags += ' ' + extra_cmake_flags
114
+ full_cmd.append(flags)
104
115
  subprocess.run(full_cmd, cwd=build_dir, check=True, env=env_vars)
105
116
  self.announce("Done building native extension.")
106
117
  self.announce("Copying native extension...")
@@ -0,0 +1,262 @@
1
+ #!/usr/bin/env python3
2
+ """Split debug symbols out of built Python wheels and repack stripped wheels.
3
+
4
+ Used by .github/workflows/python-wheel-workflow.yml after cibuildwheel finishes:
5
+
6
+ Linux: objcopy --only-keep-debug / --strip-debug / --add-gnu-debuglink
7
+ (mirrors the CMake POST_BUILD recipe in tools/python_api/CMakeLists.txt,
8
+ but runs on the final repaired wheel so auditwheel/delocate edits are
9
+ already baked in). If the wheel already ships a .debug file produced by
10
+ the in-build split and the .so is already stripped, that file is reused.
11
+ macOS: dsymutil into a .dSYM bundle + strip -S, regenerated from the final
12
+ wheel binary so it matches the shipped (delocated) .so.
13
+ Windows: move *.pdb (plus *.exp/*.lib byproducts) out of the wheel; the .pyd
14
+ itself carries no debug data when linked with /DEBUG.
15
+
16
+ Every repacked wheel gets a correct RECORD (hashes/sizes recomputed), and all
17
+ extracted symbol files land in --symbols-dir for upload as separate artifacts.
18
+ """
19
+
20
+ import argparse
21
+ import base64
22
+ import hashlib
23
+ import os
24
+ import re
25
+ import shutil
26
+ import subprocess
27
+ import sys
28
+ import tempfile
29
+ import zipfile
30
+ from pathlib import Path
31
+
32
+
33
+ def _has_debug_sections(shared_obj: Path) -> bool:
34
+ """Best-effort check for DWARF debug sections (False if unknown)."""
35
+ readelf = shutil.which("readelf")
36
+ if readelf is None:
37
+ return True # Unknown: assume symbols present so we attempt a split.
38
+ try:
39
+ out = subprocess.run(
40
+ [readelf, "-S", str(shared_obj)],
41
+ capture_output=True,
42
+ text=True,
43
+ check=False,
44
+ )
45
+ return ".debug_info" in out.stdout
46
+ except OSError:
47
+ return True
48
+
49
+
50
+ def _run(cmd: list[str], cwd: Path | None = None) -> None:
51
+ print(f"+ {' '.join(cmd)}" + (f" (cwd={cwd})" if cwd else ""), flush=True)
52
+ subprocess.run(cmd, check=True, cwd=cwd)
53
+
54
+
55
+ def _hash_record_entry(data: bytes) -> str:
56
+ digest = base64.urlsafe_b64encode(hashlib.sha256(data).digest()).rstrip(b"=")
57
+ return f"sha256={digest.decode()}"
58
+
59
+
60
+ def _repack_wheel(unpacked_dir: Path, wheel_path: Path) -> None:
61
+ """Rewrite wheel zip from unpacked_dir, dropping stale RECORD entries and
62
+ recomputing hashes for every file (required after stripping binaries)."""
63
+ record_file = None
64
+ for dist_info in unpacked_dir.glob("*.dist-info"):
65
+ candidate = dist_info / "RECORD"
66
+ if candidate.is_file():
67
+ record_file = candidate
68
+ break
69
+ if record_file is None:
70
+ raise FileNotFoundError(f"No RECORD found in unpacked wheel at {unpacked_dir}")
71
+
72
+ record_rel = record_file.relative_to(unpacked_dir).as_posix()
73
+ rows: list[str] = []
74
+ with tempfile.NamedTemporaryFile(delete=False, suffix=".whl") as tmp:
75
+ tmp_path = Path(tmp.name)
76
+ try:
77
+ with zipfile.ZipFile(tmp_path, "w", zipfile.ZIP_DEFLATED) as zf:
78
+ for root, _, files in os.walk(unpacked_dir):
79
+ for name in sorted(files):
80
+ full = Path(root) / name
81
+ rel = full.relative_to(unpacked_dir).as_posix()
82
+ if rel == record_rel:
83
+ continue
84
+ data = full.read_bytes()
85
+ zf.writestr(rel, data)
86
+ rows.append(f"{rel},{_hash_record_entry(data)},{len(data)}")
87
+ rows.append(f"{record_rel},,")
88
+ zf.writestr(record_rel, "\n".join(rows) + "\n")
89
+ shutil.move(str(tmp_path), str(wheel_path))
90
+ finally:
91
+ if tmp_path.exists():
92
+ tmp_path.unlink()
93
+
94
+
95
+ def _find_extensions(tree: Path, suffixes: tuple[str, ...]) -> list[Path]:
96
+ hits = []
97
+ for suffix in suffixes:
98
+ hits.extend(tree.rglob(f"_lbug*{suffix}"))
99
+ # Skip files that live inside symbol bundles/dirs themselves. NOTE: a
100
+ # dSYM bundle dir is named "<binary>.dSYM", so ".dSYM" is a suffix of
101
+ # a path part, not a standalone part -- check endswith, not membership.
102
+ return [
103
+ p
104
+ for p in hits
105
+ if p.is_file()
106
+ and not any(part.endswith(".dSYM") for part in p.parts)
107
+ and not p.name.endswith(".debug")
108
+ ]
109
+
110
+
111
+ def _mach_o_has_dwarf(shared_obj: Path) -> bool:
112
+ """Best-effort check for a __DWARF segment (False if unknown is unsafe).
113
+ Returns True when unsure so callers attempt a split rather than reuse."""
114
+ otool = shutil.which("otool")
115
+ if otool is None:
116
+ return True
117
+ try:
118
+ out = subprocess.run(
119
+ [otool, "-l", str(shared_obj)],
120
+ capture_output=True,
121
+ text=True,
122
+ check=False,
123
+ )
124
+ return "__DWARF" in out.stdout
125
+ except OSError:
126
+ return True
127
+
128
+
129
+ def _split_linux(wheel_path: Path, tree: Path, symbols_dir: Path) -> None:
130
+ objcopy = shutil.which("objcopy")
131
+ if objcopy is None:
132
+ print("objcopy not found; leaving Linux wheel unstripped.", flush=True)
133
+ return
134
+ for so in _find_extensions(tree, (".so",)):
135
+ stem = f"{wheel_path.stem}-{so.name}"
136
+ shipped = list(tree.rglob("*.debug"))
137
+ if shipped and not _has_debug_sections(so):
138
+ # In-build CMake split already stripped this binary; reuse it.
139
+ for dbg in shipped:
140
+ dest = symbols_dir / f"{stem}.debug"
141
+ print(f"Reusing shipped debug file {dbg} -> {dest}", flush=True)
142
+ shutil.move(str(dbg), str(dest))
143
+ continue
144
+ # Regenerate from the final (repaired) binary.
145
+ for dbg in shipped:
146
+ dbg.unlink()
147
+ dbg_out = so.parent / f"{so.name}.debug"
148
+ _run([objcopy, "--only-keep-debug", str(so), str(dbg_out)])
149
+ _run([objcopy, "--strip-debug", str(so)])
150
+ # --strip-debug leaves a pre-existing .gnu_debuglink behind, and
151
+ # --add-gnu-debuglink refuses to overwrite it, so drop it first.
152
+ subprocess.run(
153
+ [objcopy, "-R", ".gnu_debuglink", str(so)],
154
+ capture_output=True,
155
+ check=False,
156
+ )
157
+ # NOTE: --add-gnu-debuglink resolves a relative filename against the
158
+ # process CWD (not the binary's dir), so run from so.parent so the
159
+ # stored link stays a bare filename while objcopy can read the file.
160
+ _run([objcopy, f"--add-gnu-debuglink={dbg_out.name}", str(so)], cwd=so.parent)
161
+ dest = symbols_dir / f"{stem}.debug"
162
+ print(f"Moving {dbg_out} -> {dest}", flush=True)
163
+ shutil.move(str(dbg_out), str(dest))
164
+
165
+
166
+ def _split_macos(wheel_path: Path, tree: Path, symbols_dir: Path) -> None:
167
+ dsymutil = shutil.which("dsymutil")
168
+ strip = shutil.which("strip")
169
+ if dsymutil is None or strip is None:
170
+ print("dsymutil/strip not found; leaving macOS wheel unstripped.", flush=True)
171
+ return
172
+ for so in _find_extensions(tree, (".so",)):
173
+ stem = f"{wheel_path.stem}-{so.name}"
174
+ shipped = [d for d in tree.rglob("*.dSYM")]
175
+ if shipped and not _mach_o_has_dwarf(so):
176
+ # In-build split already stripped this binary; reuse its bundle.
177
+ for dsym in shipped:
178
+ dest = symbols_dir / f"{stem}.dSYM"
179
+ if dest.exists():
180
+ shutil.rmtree(dest) if dest.is_dir() else dest.unlink()
181
+ print(f"Reusing shipped dSYM {dsym} -> {dest}", flush=True)
182
+ shutil.move(str(dsym), str(dest))
183
+ continue
184
+ # Otherwise drop any stale pre-repair bundle and regenerate below so
185
+ # the dSYM matches the final delocated binary.
186
+ for stale in shipped:
187
+ if stale.is_dir():
188
+ shutil.rmtree(stale)
189
+ elif stale.is_file():
190
+ stale.unlink()
191
+ dsym_out = Path(tempfile.mkdtemp()) / f"{so.name}.dSYM"
192
+ _run([dsymutil, str(so), "-o", str(dsym_out)])
193
+ _run([strip, "-S", str(so)])
194
+ dest = symbols_dir / f"{stem}.dSYM"
195
+ if dest.exists():
196
+ shutil.rmtree(dest)
197
+ print(f"Moving {dsym_out} -> {dest}", flush=True)
198
+ shutil.move(str(dsym_out), str(dest))
199
+
200
+
201
+ def _split_windows(wheel_path: Path, tree: Path, symbols_dir: Path) -> None:
202
+ stem = wheel_path.stem
203
+ for pattern in ("*.pdb", "*.exp", "*.lib"):
204
+ for artifact in tree.rglob(pattern):
205
+ if not artifact.is_file():
206
+ continue
207
+ if pattern == "*.pdb":
208
+ dest = symbols_dir / f"{stem}-{artifact.name}"
209
+ print(f"Moving {artifact} -> {dest}", flush=True)
210
+ shutil.move(str(artifact), str(dest))
211
+ else:
212
+ # Linker byproducts: never ship in the wheel.
213
+ print(f"Removing linker byproduct {artifact}", flush=True)
214
+ artifact.unlink()
215
+ # The .pyd linked with /DEBUG already carries no debug data; nothing to strip.
216
+
217
+
218
+ def process_wheel(wheel_path: Path, symbols_dir: Path, os_name: str) -> None:
219
+ print(f"=== {wheel_path.name} ({os_name}) ===", flush=True)
220
+ with tempfile.TemporaryDirectory() as tmp:
221
+ tree = Path(tmp) / "wheel"
222
+ with zipfile.ZipFile(wheel_path, "r") as zf:
223
+ zf.extractall(tree)
224
+ if os_name == "linux":
225
+ _split_linux(wheel_path, tree, symbols_dir)
226
+ elif os_name == "macos":
227
+ _split_macos(wheel_path, tree, symbols_dir)
228
+ elif os_name == "windows":
229
+ _split_windows(wheel_path, tree, symbols_dir)
230
+ else:
231
+ raise ValueError(f"Unknown os: {os_name}")
232
+ _repack_wheel(tree, wheel_path)
233
+ print(f"Repacked stripped wheel: {wheel_path.name}", flush=True)
234
+
235
+
236
+ def main() -> int:
237
+ parser = argparse.ArgumentParser(description=__doc__)
238
+ parser.add_argument("--wheel-dir", required=True, help="Directory with *.whl files")
239
+ parser.add_argument("--symbols-dir", required=True, help="Output dir for symbols")
240
+ parser.add_argument("--os", required=True, choices=["linux", "macos", "windows"])
241
+ args = parser.parse_args()
242
+
243
+ wheel_dir = Path(args.wheel_dir)
244
+ symbols_dir = Path(args.symbols_dir)
245
+ symbols_dir.mkdir(parents=True, exist_ok=True)
246
+
247
+ wheels = sorted(wheel_dir.glob("*.whl"))
248
+ if not wheels:
249
+ print(f"No wheels found in {wheel_dir}; nothing to do.", flush=True)
250
+ return 0
251
+ for wheel in wheels:
252
+ process_wheel(wheel, symbols_dir, args.os)
253
+
254
+ produced = sorted(p.name for p in symbols_dir.iterdir())
255
+ print(f"Symbol files produced ({len(produced)}):", flush=True)
256
+ for name in produced:
257
+ print(f" {name}", flush=True)
258
+ return 0
259
+
260
+
261
+ if __name__ == "__main__":
262
+ sys.exit(main())
@@ -57,7 +57,7 @@ iC_ExportDatabase
57
57
  : EXPORT SP DATABASE SP StringLiteral ( SP? '(' SP? iC_Options SP? ')' )? ;
58
58
 
59
59
  iC_ImportDatabase
60
- : IMPORT SP DATABASE SP StringLiteral;
60
+ : IMPORT SP DATABASE SP StringLiteral ( SP? '(' SP? iC_Options SP? ')' )? ;
61
61
 
62
62
  iC_AttachDatabase
63
63
  : ATTACH SP StringLiteral (SP AS SP oC_SchemaName)? SP '(' SP? DBTYPE SP oC_SymbolicName (SP? ',' SP? iC_Options)? SP? ')' ;
@@ -11,6 +11,7 @@
11
11
  #include "parser/parser.h"
12
12
  #include "parser/port_db.h"
13
13
  #include "parser/query/regular_query.h"
14
+ #include "storage/table/ice_disk_constants.h"
14
15
  #include "transaction/transaction.h"
15
16
  #include <format>
16
17
 
@@ -35,6 +36,10 @@ FileTypeInfo getFileType(case_insensitive_map_t<Value>& options) {
35
36
  auto valueStr = value.getValue<std::string>();
36
37
  StringUtils::toUpper(valueStr);
37
38
  fileTypeInfo = FileTypeInfo{FileTypeUtils::fromString(valueStr), valueStr};
39
+ if (fileTypeInfo.fileType != FileType::PARQUET) {
40
+ throw BinderException(
41
+ "Export database only supports the parquet (icebug-disk CSR) format.");
42
+ }
38
43
  options.erase(PortDBConstants::EXPORT_FORMAT_OPTION);
39
44
  }
40
45
  return fileTypeInfo;
@@ -57,66 +62,134 @@ void bindExportTableData(ExportedTableData& tableData, const std::string& query,
57
62
  tableData.regularQuery = std::move(boundQuery);
58
63
  }
59
64
 
65
+ static std::string getPropertyReturnList(const TableCatalogEntry& entry, const std::string& var) {
66
+ std::string props;
67
+ for (auto& property : entry.getProperties()) {
68
+ if (property.getType() == LogicalType::INTERNAL_ID()) {
69
+ continue;
70
+ }
71
+ props += std::format(",{}.{} as {}", var, StringUtils::quoteIdentifier(property.getName()),
72
+ StringUtils::quoteIdentifier(property.getName()));
73
+ }
74
+ return props;
75
+ }
76
+
60
77
  static std::string getExportNodeTableDataQuery(const TableCatalogEntry& entry) {
61
- return std::format("match (a:{}) return a.*", StringUtils::quoteIdentifier(entry.getName()));
78
+ // Order by storage offset so that row i in nodes_<table>.parquet holds the node
79
+ // with dense offset i, matching the CSR target/ptr values in the rel tables.
80
+ // Properties are aliased explicitly so parquet columns carry clean names.
81
+ auto props = getPropertyReturnList(entry, "a");
82
+ auto returns = props.empty() ? "a.*" : props.substr(1);
83
+ return std::format("match (a:{}) return {} order by offset(id(a))",
84
+ StringUtils::quoteIdentifier(entry.getName()), returns);
62
85
  }
63
86
 
64
- static std::string getExportRelTableDataQuery(const TableCatalogEntry& relGroupEntry,
87
+ static std::string getExportRelIndicesQuery(const TableCatalogEntry& relGroupEntry,
65
88
  const NodeTableCatalogEntry& srcEntry, const NodeTableCatalogEntry& dstEntry) {
66
- return std::format("match (a:{})-[r:{}]->(b:{}) return a.{},b.{},r.*;",
89
+ // CSR indices table: dense target offsets (+ edge properties), ordered by
90
+ // (source offset, target offset). The source offset is encoded by indptr.
91
+ return std::format("match (a:{})-[r:{}]->(b:{}) return offset(id(b)) as target{} "
92
+ "order by offset(id(a)), offset(id(b));",
67
93
  StringUtils::quoteIdentifier(srcEntry.getName()),
68
94
  StringUtils::quoteIdentifier(relGroupEntry.getName()),
69
95
  StringUtils::quoteIdentifier(dstEntry.getName()),
70
- StringUtils::quoteIdentifier(srcEntry.getPrimaryKeyName()),
71
- StringUtils::quoteIdentifier(dstEntry.getPrimaryKeyName()));
96
+ getPropertyReturnList(relGroupEntry, "r"));
97
+ }
98
+
99
+ static std::string getExportRelIndptrQuery(const TableCatalogEntry& relGroupEntry,
100
+ const NodeTableCatalogEntry& srcEntry, const NodeTableCatalogEntry& dstEntry) {
101
+ // One row per source node (OPTIONAL MATCH keeps zero-degree nodes), consumed by
102
+ // the indptr export function which sorts by src and prefix-sums the degrees.
103
+ return std::format(
104
+ "match (a:{}) optional match (a)-[r:{}]->(b:{}) return offset(id(a)) as src, "
105
+ "count(r) as degree;",
106
+ StringUtils::quoteIdentifier(srcEntry.getName()),
107
+ StringUtils::quoteIdentifier(relGroupEntry.getName()),
108
+ StringUtils::quoteIdentifier(dstEntry.getName()));
109
+ }
110
+
111
+ static std::string getExportRelFlatQuery(const TableCatalogEntry& relGroupEntry,
112
+ const NodeTableCatalogEntry& srcEntry, const NodeTableCatalogEntry& dstEntry) {
113
+ // Flat relationship file for native re-import: source/target primary-key values
114
+ // (COPY FROM resolves them back to node offsets) followed by edge properties.
115
+ return std::format("match (a:{})-[r:{}]->(b:{}) return {}.{} as {}, {}.{} as {}{};",
116
+ StringUtils::quoteIdentifier(srcEntry.getName()),
117
+ StringUtils::quoteIdentifier(relGroupEntry.getName()),
118
+ StringUtils::quoteIdentifier(dstEntry.getName()), "a",
119
+ StringUtils::quoteIdentifier(srcEntry.getPrimaryKeyDefinition().getName()),
120
+ StringUtils::quoteIdentifier("from"), "b",
121
+ StringUtils::quoteIdentifier(dstEntry.getPrimaryKeyDefinition().getName()),
122
+ StringUtils::quoteIdentifier("to"), getPropertyReturnList(relGroupEntry, "r"));
72
123
  }
73
124
 
74
125
  static std::vector<ExportedTableData> getExportInfo(const Catalog& catalog,
75
- main::ClientContext* context, Binder* binder, FileTypeInfo& fileTypeInfo) {
126
+ main::ClientContext* context, Binder* binder, [[maybe_unused]] FileTypeInfo& fileTypeInfo) {
76
127
  auto transaction = Transaction::Get(*context);
77
128
  std::vector<ExportedTableData> exportData;
78
129
  for (auto entry : catalog.getNodeTableEntries(transaction, false /*useInternal*/)) {
79
130
  ExportedTableData tableData;
80
131
  tableData.tableName = entry->getName();
81
- tableData.fileName =
82
- entry->getName() + "." + StringUtils::getLower(fileTypeInfo.fileTypeStr);
132
+ tableData.fileName = std::format("nodes_{}.parquet", entry->getName());
83
133
  auto query = getExportNodeTableDataQuery(*entry);
84
134
  bindExportTableData(tableData, query, context, binder);
85
135
  exportData.push_back(std::move(tableData));
86
136
  }
87
137
  for (auto entry : catalog.getRelGroupEntries(transaction, false /* useInternal */)) {
88
138
  auto& relGroupEntry = entry->constCast<RelGroupCatalogEntry>();
139
+ if (relGroupEntry.getRelEntryInfos().size() != 1) {
140
+ throw BinderException(std::format(
141
+ "Export database: rel table {} connects multiple node pairs, which is not "
142
+ "supported for icebug-disk CSR export yet.",
143
+ relGroupEntry.getName()));
144
+ }
89
145
  for (auto& info : relGroupEntry.getRelEntryInfos()) {
90
- ExportedTableData tableData;
91
146
  auto srcTableID = info.nodePair.srcTableID;
92
147
  auto dstTableID = info.nodePair.dstTableID;
93
148
  auto& srcEntry = catalog.getTableCatalogEntry(transaction, srcTableID)
94
149
  ->constCast<NodeTableCatalogEntry>();
95
150
  auto& dstEntry = catalog.getTableCatalogEntry(transaction, dstTableID)
96
151
  ->constCast<NodeTableCatalogEntry>();
97
- tableData.tableName = entry->getName();
98
- tableData.fileName =
99
- std::format("{}_{}_{}.{}", relGroupEntry.getName(), srcEntry.getName(),
100
- dstEntry.getName(), StringUtils::getLower(fileTypeInfo.fileTypeStr));
101
- auto query = getExportRelTableDataQuery(relGroupEntry, srcEntry, dstEntry);
102
- bindExportTableData(tableData, query, context, binder);
103
- exportData.push_back(std::move(tableData));
152
+ ExportedTableData indicesData;
153
+ indicesData.tableName = entry->getName();
154
+ indicesData.fileName = std::format("indices_{}.parquet", relGroupEntry.getName());
155
+ bindExportTableData(indicesData,
156
+ getExportRelIndicesQuery(relGroupEntry, srcEntry, dstEntry), context, binder);
157
+ exportData.push_back(std::move(indicesData));
158
+ ExportedTableData indptrData;
159
+ indptrData.tableName = entry->getName();
160
+ indptrData.fileName = std::format("indptr_{}.parquet", relGroupEntry.getName());
161
+ indptrData.isIndptr = true;
162
+ bindExportTableData(indptrData,
163
+ getExportRelIndptrQuery(relGroupEntry, srcEntry, dstEntry), context, binder);
164
+ exportData.push_back(std::move(indptrData));
165
+ // Flat relationship file (from/to primary-key values + properties), used by
166
+ // IMPORT DATABASE to ingest the relationship table into native storage. It is
167
+ // ignored when the export is mounted as icebug-disk (which uses the CSR files).
168
+ ExportedTableData flatRelData;
169
+ flatRelData.tableName = entry->getName();
170
+ flatRelData.fileName = std::format("rels_{}.parquet", relGroupEntry.getName());
171
+ bindExportTableData(flatRelData,
172
+ getExportRelFlatQuery(relGroupEntry, srcEntry, dstEntry), context, binder);
173
+ exportData.push_back(std::move(flatRelData));
104
174
  }
105
175
  }
106
176
 
177
+ // Index aux tables (e.g. the FTS internal stopwords table) are not part of the CSR
178
+ // layout, but index.cypher needs them on import when the original stopwords source
179
+ // table no longer exists. Export them as <tableName>.parquet; the FTS index's
180
+ // toCypher references the file path in that case.
107
181
  for (auto indexEntry : catalog.getIndexEntries(transaction)) {
108
- // Export
109
- ExportedTableData tableData;
110
- auto entry = indexEntry->getTableEntryToExport(context);
111
- if (entry == nullptr) {
182
+ auto auxEntry = indexEntry->getTableEntryToExport(context);
183
+ if (auxEntry == nullptr) {
112
184
  continue;
113
185
  }
114
- DASSERT(entry->getTableType() == TableType::NODE);
115
- tableData.tableName = entry->getName();
116
- tableData.fileName =
117
- entry->getName() + "." + StringUtils::getLower(fileTypeInfo.fileTypeStr);
118
- auto query = getExportNodeTableDataQuery(*entry);
119
- bindExportTableData(tableData, query, context, binder);
186
+ ExportedTableData tableData;
187
+ tableData.tableName = auxEntry->getName();
188
+ tableData.fileName = std::format("{}.parquet", auxEntry->getName());
189
+ bindExportTableData(tableData,
190
+ std::format("match (a:{}) return a.*",
191
+ StringUtils::quoteIdentifier(auxEntry->getName())),
192
+ context, binder);
120
193
  exportData.push_back(std::move(tableData));
121
194
  }
122
195
  return exportData;
@@ -158,10 +231,17 @@ std::unique_ptr<BoundStatement> Binder::bindExportDatabaseClause(const Statement
158
231
  }
159
232
  auto exportSchemaOnly = schemaOnly(parsedOptions, exportDB);
160
233
  if (!exportSchemaOnly && fileTypeInfo.fileType != FileType::CSV && parsedOptions.size() != 0) {
161
- throw BinderException{"Only export to csv can have options."};
234
+ throw BinderException{"Export database does not support copy options with the parquet "
235
+ "(icebug-disk CSR) format."};
162
236
  }
163
237
  auto exportData =
164
238
  getExportInfo(*Catalog::Get(*clientContext), clientContext, this, fileTypeInfo);
239
+ // Stamp every exported parquet file with the icebug-disk format version so that
240
+ // IceDiskUtils::checkVersionCompatibility can validate it when the export is mounted.
241
+ if (fileTypeInfo.fileType == FileType::PARQUET && !exportSchemaOnly) {
242
+ parsedOptions.insert_or_assign(PortDBConstants::ICEBUG_DISK_VERSION_OPTION,
243
+ Value{std::string{storage::IceDiskConstants::CURRENT_VERSION}});
244
+ }
165
245
  auto boundFilePath = VirtualFileSystem::GetUnsafe(*clientContext)
166
246
  ->expandPath(clientContext, exportDB.getFilePath());
167
247
  return std::make_unique<BoundExportDatabase>(boundFilePath, fileTypeInfo, std::move(exportData),