ladybug 0.20.2__tar.gz → 0.20.2.dev20260908__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sdist/PKG-INFO +1 -1
- sdist/ladybug-source/CMakeLists.txt +2 -2
- sdist/ladybug-source/docs/multi_parent_lifetime.md +12 -0
- sdist/ladybug-source/scripts/antlr4/Cypher.g4 +1 -1
- sdist/ladybug-source/scripts/antlr4/hash.md5 +1 -1
- sdist/ladybug-source/scripts/pip-package/setup.py +13 -2
- sdist/ladybug-source/scripts/pip-package/split_debug_symbols.py +262 -0
- sdist/ladybug-source/src/antlr4/Cypher.g4 +1 -1
- sdist/ladybug-source/src/binder/bind/bind_export_database.cpp +107 -27
- sdist/ladybug-source/src/binder/bind/bind_import_database.cpp +100 -1
- sdist/ladybug-source/src/binder/query/query_graph.cpp +15 -8
- sdist/ladybug-source/src/catalog/catalog_entry/node_table_catalog_entry.cpp +3 -0
- sdist/ladybug-source/src/catalog/catalog_entry/rel_group_catalog_entry.cpp +5 -1
- sdist/ladybug-source/src/function/export/CMakeLists.txt +2 -1
- sdist/ladybug-source/src/function/export/export_indptr_function.cpp +171 -0
- sdist/ladybug-source/src/function/export/export_parquet_function.cpp +18 -2
- sdist/ladybug-source/src/include/binder/bound_export_database.h +3 -0
- sdist/ladybug-source/src/include/common/constants.h +5 -0
- sdist/ladybug-source/src/include/common/data_chunk/data_chunk_state.h +34 -54
- sdist/ladybug-source/src/include/common/types/types.h +5 -1
- sdist/ladybug-source/src/include/function/export/export_function.h +8 -0
- sdist/ladybug-source/src/include/function/hash/hash_functions.h +6 -1
- sdist/ladybug-source/src/include/main/client_config.h +26 -0
- sdist/ladybug-source/src/include/main/client_context.h +3 -0
- sdist/ladybug-source/src/include/main/settings.h +10 -0
- sdist/ladybug-source/src/include/optimizer/count_rel_table_optimizer.h +12 -0
- sdist/ladybug-source/src/include/optimizer/group_key_predicate_push_down_optimizer.h +49 -0
- sdist/ladybug-source/src/include/parser/expression/parsed_subquery_expression.h +1 -0
- sdist/ladybug-source/src/include/parser/port_db.h +3 -0
- sdist/ladybug-source/src/include/planner/operator/logical_operator.h +2 -0
- sdist/ladybug-source/src/include/planner/operator/scan/logical_count_anti_edge_chain.h +116 -0
- sdist/ladybug-source/src/include/planner/operator/scan/logical_count_extend_chain.h +108 -0
- sdist/ladybug-source/src/include/processor/operator/aggregate/base_aggregate.h +2 -0
- sdist/ladybug-source/src/include/processor/operator/aggregate/hash_aggregate.h +6 -0
- sdist/ladybug-source/src/include/processor/operator/base_partitioner_shared_state.h +9 -2
- sdist/ladybug-source/src/include/processor/operator/persistent/writer/parquet/parquet_writer.h +3 -1
- sdist/ladybug-source/src/include/processor/operator/physical_operator.h +2 -0
- sdist/ladybug-source/src/include/processor/operator/scan/count_anti_edge_chain.h +114 -0
- sdist/ladybug-source/src/include/processor/operator/scan/count_extend_chain.h +109 -0
- sdist/ladybug-source/src/include/processor/operator/scan/scan_node_table.h +16 -1
- sdist/ladybug-source/src/include/processor/operator/scan/scan_rel_table.h +21 -9
- sdist/ladybug-source/src/include/processor/plan_mapper.h +4 -0
- sdist/ladybug-source/src/include/processor/processor.h +38 -0
- sdist/ladybug-source/src/include/processor/result/result_set_descriptor.h +2 -2
- sdist/ladybug-source/src/include/storage/file_handle.h +3 -0
- sdist/ladybug-source/src/include/storage/index/index.h +10 -0
- sdist/ladybug-source/src/include/storage/storage_version_info.h +1 -1
- sdist/ladybug-source/src/include/storage/table/arrow_node_table.h +6 -1
- sdist/ladybug-source/src/include/storage/table/arrow_rel_table.h +4 -4
- sdist/ladybug-source/src/include/storage/table/arrow_table_support.h +18 -5
- sdist/ladybug-source/src/include/storage/table/csr_node_group.h +8 -0
- sdist/ladybug-source/src/include/storage/table/rel_table.h +33 -0
- sdist/ladybug-source/src/include/storage/table/table.h +6 -0
- sdist/ladybug-source/src/main/client_context.cpp +15 -0
- sdist/ladybug-source/src/main/db_config.cpp +2 -1
- sdist/ladybug-source/src/main/settings.cpp +46 -0
- sdist/ladybug-source/src/optimizer/CMakeLists.txt +1 -0
- sdist/ladybug-source/src/optimizer/count_rel_table_optimizer.cpp +675 -0
- sdist/ladybug-source/src/optimizer/filter_push_down_optimizer.cpp +6 -1
- sdist/ladybug-source/src/optimizer/group_key_predicate_push_down_optimizer.cpp +204 -0
- sdist/ladybug-source/src/optimizer/optimizer.cpp +162 -0
- sdist/ladybug-source/src/parser/expression/parsed_expression_visitor.cpp +23 -0
- sdist/ladybug-source/src/parser/transform/transform_port_db.cpp +5 -1
- sdist/ladybug-source/src/planner/operator/logical_operator.cpp +8 -2
- sdist/ladybug-source/src/planner/operator/scan/CMakeLists.txt +2 -0
- sdist/ladybug-source/src/planner/operator/scan/logical_count_anti_edge_chain.cpp +20 -0
- sdist/ladybug-source/src/planner/operator/scan/logical_count_extend_chain.cpp +21 -0
- sdist/ladybug-source/src/planner/plan/plan_join_order.cpp +54 -24
- sdist/ladybug-source/src/planner/plan/plan_port_db.cpp +6 -1
- sdist/ladybug-source/src/processor/map/CMakeLists.txt +2 -0
- sdist/ladybug-source/src/processor/map/map_aggregate.cpp +11 -1
- sdist/ladybug-source/src/processor/map/map_count_anti_edge_chain.cpp +52 -0
- sdist/ladybug-source/src/processor/map/map_count_extend_chain.cpp +52 -0
- sdist/ladybug-source/src/processor/map/map_merge.cpp +2 -2
- sdist/ladybug-source/src/processor/map/plan_mapper.cpp +6 -0
- sdist/ladybug-source/src/processor/operator/aggregate/hash_aggregate.cpp +10 -3
- sdist/ladybug-source/src/processor/operator/aggregate/packed_filtered_count.cpp +44 -6
- sdist/ladybug-source/src/processor/operator/base_partitioner_shared_state.cpp +5 -3
- sdist/ladybug-source/src/processor/operator/ddl/drop.cpp +4 -1
- sdist/ladybug-source/src/processor/operator/persistent/writer/parquet/parquet_writer.cpp +7 -2
- sdist/ladybug-source/src/processor/operator/physical_operator.cpp +4 -0
- sdist/ladybug-source/src/processor/operator/scan/CMakeLists.txt +2 -0
- sdist/ladybug-source/src/processor/operator/scan/count_anti_edge_chain.cpp +349 -0
- sdist/ladybug-source/src/processor/operator/scan/count_extend_chain.cpp +193 -0
- sdist/ladybug-source/src/processor/operator/scan/scan_node_table.cpp +71 -4
- sdist/ladybug-source/src/processor/operator/scan/scan_rel_table.cpp +22 -35
- sdist/ladybug-source/src/processor/operator/simple/export_db.cpp +30 -101
- sdist/ladybug-source/src/processor/processor.cpp +32 -0
- sdist/ladybug-source/src/processor/processor_task.cpp +14 -21
- sdist/ladybug-source/src/processor/result/factorized_table.cpp +7 -2
- sdist/ladybug-source/src/processor/result/pattern_creation_info_table.cpp +7 -2
- sdist/ladybug-source/src/processor/result/result_set_descriptor.cpp +2 -2
- sdist/ladybug-source/src/storage/file_handle.cpp +10 -7
- sdist/ladybug-source/src/storage/local_storage/local_node_table.cpp +12 -1
- sdist/ladybug-source/src/storage/shadow_file.cpp +43 -12
- sdist/ladybug-source/src/storage/storage_manager.cpp +12 -31
- sdist/ladybug-source/src/storage/table/arrow_node_table.cpp +13 -3
- sdist/ladybug-source/src/storage/table/arrow_rel_table.cpp +20 -7
- sdist/ladybug-source/src/storage/table/arrow_table_support.cpp +24 -20
- sdist/ladybug-source/src/storage/table/csr_node_group.cpp +93 -1
- sdist/ladybug-source/src/storage/table/ice_disk_node_table.cpp +26 -2
- sdist/ladybug-source/src/storage/table/node_group.cpp +18 -8
- sdist/ladybug-source/src/storage/table/node_table.cpp +129 -12
- sdist/ladybug-source/src/storage/table/rel_table.cpp +118 -15
- sdist/ladybug-source/src/storage/table/rel_table_data.cpp +17 -1
- sdist/ladybug-source/third_party/antlr4_cypher/cypher_parser.cpp +3323 -3272
- sdist/ladybug-source/third_party/antlr4_cypher/include/cypher_parser.h +1 -0
- sdist/ladybug-source/tools/python_api/CMakeLists.txt +81 -0
- sdist/ladybug-source/tools/python_api/src_cpp/include/pyarrow/pyarrow_scan.h +5 -0
- sdist/ladybug-source/tools/python_api/test/test_scan_pandas_pyarrow.py +12 -0
- sdist/ladybug.egg-info/PKG-INFO +1 -1
- sdist/pyproject.toml +1 -1
- sdist/setup.py +13 -2
- sdist/test/test_scan_pandas_pyarrow.py +12 -0
sdist/PKG-INFO
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
cmake_minimum_required(VERSION 3.15)
|
|
2
2
|
|
|
3
|
-
project(Lbug VERSION 0.20.2 LANGUAGES CXX C)
|
|
3
|
+
project(Lbug VERSION 0.20.2.20260908 LANGUAGES CXX C)
|
|
4
4
|
|
|
5
5
|
option(SINGLE_THREADED "Single-threaded mode" FALSE)
|
|
6
6
|
if(SINGLE_THREADED)
|
|
@@ -522,7 +522,7 @@ unset(LBUG_LINK_LIBRARIES)
|
|
|
522
522
|
|
|
523
523
|
add_definitions(-DLBUG_ROOT_DIRECTORY="${PROJECT_SOURCE_DIR}")
|
|
524
524
|
add_definitions(-DLBUG_CMAKE_VERSION="${CMAKE_PROJECT_VERSION}")
|
|
525
|
-
add_definitions(-DLBUG_EXTENSION_VERSION="
|
|
525
|
+
add_definitions(-DLBUG_EXTENSION_VERSION="dev")
|
|
526
526
|
|
|
527
527
|
if(BUILD_LBUG OR LBUG_API_USE_PRECOMPILED_LIB)
|
|
528
528
|
include_directories(
|
|
@@ -1,5 +1,17 @@
|
|
|
1
1
|
# Multi-parent packed child slices: lifetime & representation
|
|
2
2
|
|
|
3
|
+
> **Status update (multi-parent scan implemented):** the deferred change described below has
|
|
4
|
+
> been implemented. `PackedChildSlices` now holds a `shared_ptr<SelectionVector>` aliasing the
|
|
5
|
+
> bound (parent) chunk state's selection vector plus a prefix-sum `offsets` over ALL served
|
|
6
|
+
> parents (zeros allowed, zero-length ranges skipped by consumers). The CSR scan packs children
|
|
7
|
+
> of multiple parents per output batch in `CSRNodeGroupScanState::tryScanCachedTuplesPacked`
|
|
8
|
+
> (persistent with-cache path only). Multi-parent batches are enabled per-consumer:
|
|
9
|
+
> `ScanRelTable::setMultiParentPackedScanEnabled(true)` is set by the plan mapper only when the
|
|
10
|
+
> packed-aware `PackedFilteredCount` consumes the scan output directly. Plans that materialize
|
|
11
|
+
> the scan output across a `FactorizedTable` (e.g. when the filter predicate touches an nbr
|
|
12
|
+
> node property, which routes the property fetch through a hash join) keep the
|
|
13
|
+
> one-parent-per-batch contract — exactly the materialization hazard described below.
|
|
14
|
+
|
|
3
15
|
Context: the `enable_packed_path_extend` path reuses the adjacency-list (CSR)
|
|
4
16
|
rel scan to implement a physical packed extend. `ScanRelTable` attaches a
|
|
5
17
|
`PackedChildSlices` descriptor to the **output** `DataChunkState`
|
|
@@ -326,7 +326,7 @@ iC_ExportDatabase
|
|
|
326
326
|
: EXPORT SP DATABASE SP StringLiteral ( SP? '(' SP? iC_Options SP? ')' )? ;
|
|
327
327
|
|
|
328
328
|
iC_ImportDatabase
|
|
329
|
-
: IMPORT SP DATABASE SP StringLiteral;
|
|
329
|
+
: IMPORT SP DATABASE SP StringLiteral ( SP? '(' SP? iC_Options SP? ')' )? ;
|
|
330
330
|
|
|
331
331
|
iC_AttachDatabase
|
|
332
332
|
: ATTACH SP StringLiteral (SP AS SP oC_SchemaName)? SP '(' SP? DBTYPE SP oC_SymbolicName (SP? ',' SP? iC_Options)? SP? ')' ;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
|
|
1
|
+
6fc9fc9085d4f1028a4fb2b319152cb5
|
|
@@ -91,16 +91,27 @@ class CMakeBuild(build_ext):
|
|
|
91
91
|
if os.path.exists(flags_file):
|
|
92
92
|
with open(flags_file) as f:
|
|
93
93
|
extra_cmake_flags = f.read().strip()
|
|
94
|
+
# Wheel builds must not split symbols at build time: raw .debug/dSYM
|
|
95
|
+
# artifacts packaged inside the wheel break auditwheel/delocate
|
|
96
|
+
# repair. The split happens on the final repaired wheel instead (see
|
|
97
|
+
# split_debug_symbols.py), which needs the debug info to still be
|
|
98
|
+
# present in the shipped binary.
|
|
99
|
+
no_split_flag = '-DLBUG_SPLIT_DEBUG_SYMBOLS=OFF'
|
|
94
100
|
if precompiled_lib_path:
|
|
95
101
|
self.announce("Using precompiled liblbug from %s." % precompiled_lib_path)
|
|
96
102
|
flags = (
|
|
97
|
-
'EXTRA_CMAKE_FLAGS
|
|
103
|
+
'EXTRA_CMAKE_FLAGS=%s -DBUILD_LBUG=FALSE -DBUILD_SHELL=FALSE '
|
|
98
104
|
'-DLBUG_API_USE_PRECOMPILED_LIB=TRUE '
|
|
99
|
-
'-DLBUG_API_PRECOMPILED_LIB_PATH=%s' % precompiled_lib_path
|
|
105
|
+
'-DLBUG_API_PRECOMPILED_LIB_PATH=%s' % (no_split_flag, precompiled_lib_path)
|
|
100
106
|
)
|
|
101
107
|
if extra_cmake_flags:
|
|
102
108
|
flags += ' ' + extra_cmake_flags
|
|
103
109
|
full_cmd.append(flags)
|
|
110
|
+
else:
|
|
111
|
+
flags = 'EXTRA_CMAKE_FLAGS=%s' % no_split_flag
|
|
112
|
+
if extra_cmake_flags:
|
|
113
|
+
flags += ' ' + extra_cmake_flags
|
|
114
|
+
full_cmd.append(flags)
|
|
104
115
|
subprocess.run(full_cmd, cwd=build_dir, check=True, env=env_vars)
|
|
105
116
|
self.announce("Done building native extension.")
|
|
106
117
|
self.announce("Copying native extension...")
|
|
@@ -0,0 +1,262 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Split debug symbols out of built Python wheels and repack stripped wheels.
|
|
3
|
+
|
|
4
|
+
Used by .github/workflows/python-wheel-workflow.yml after cibuildwheel finishes:
|
|
5
|
+
|
|
6
|
+
Linux: objcopy --only-keep-debug / --strip-debug / --add-gnu-debuglink
|
|
7
|
+
(mirrors the CMake POST_BUILD recipe in tools/python_api/CMakeLists.txt,
|
|
8
|
+
but runs on the final repaired wheel so auditwheel/delocate edits are
|
|
9
|
+
already baked in). If the wheel already ships a .debug file produced by
|
|
10
|
+
the in-build split and the .so is already stripped, that file is reused.
|
|
11
|
+
macOS: dsymutil into a .dSYM bundle + strip -S, regenerated from the final
|
|
12
|
+
wheel binary so it matches the shipped (delocated) .so.
|
|
13
|
+
Windows: move *.pdb (plus *.exp/*.lib byproducts) out of the wheel; the .pyd
|
|
14
|
+
itself carries no debug data when linked with /DEBUG.
|
|
15
|
+
|
|
16
|
+
Every repacked wheel gets a correct RECORD (hashes/sizes recomputed), and all
|
|
17
|
+
extracted symbol files land in --symbols-dir for upload as separate artifacts.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
import argparse
|
|
21
|
+
import base64
|
|
22
|
+
import hashlib
|
|
23
|
+
import os
|
|
24
|
+
import re
|
|
25
|
+
import shutil
|
|
26
|
+
import subprocess
|
|
27
|
+
import sys
|
|
28
|
+
import tempfile
|
|
29
|
+
import zipfile
|
|
30
|
+
from pathlib import Path
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def _has_debug_sections(shared_obj: Path) -> bool:
|
|
34
|
+
"""Best-effort check for DWARF debug sections (False if unknown)."""
|
|
35
|
+
readelf = shutil.which("readelf")
|
|
36
|
+
if readelf is None:
|
|
37
|
+
return True # Unknown: assume symbols present so we attempt a split.
|
|
38
|
+
try:
|
|
39
|
+
out = subprocess.run(
|
|
40
|
+
[readelf, "-S", str(shared_obj)],
|
|
41
|
+
capture_output=True,
|
|
42
|
+
text=True,
|
|
43
|
+
check=False,
|
|
44
|
+
)
|
|
45
|
+
return ".debug_info" in out.stdout
|
|
46
|
+
except OSError:
|
|
47
|
+
return True
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _run(cmd: list[str], cwd: Path | None = None) -> None:
|
|
51
|
+
print(f"+ {' '.join(cmd)}" + (f" (cwd={cwd})" if cwd else ""), flush=True)
|
|
52
|
+
subprocess.run(cmd, check=True, cwd=cwd)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _hash_record_entry(data: bytes) -> str:
|
|
56
|
+
digest = base64.urlsafe_b64encode(hashlib.sha256(data).digest()).rstrip(b"=")
|
|
57
|
+
return f"sha256={digest.decode()}"
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def _repack_wheel(unpacked_dir: Path, wheel_path: Path) -> None:
|
|
61
|
+
"""Rewrite wheel zip from unpacked_dir, dropping stale RECORD entries and
|
|
62
|
+
recomputing hashes for every file (required after stripping binaries)."""
|
|
63
|
+
record_file = None
|
|
64
|
+
for dist_info in unpacked_dir.glob("*.dist-info"):
|
|
65
|
+
candidate = dist_info / "RECORD"
|
|
66
|
+
if candidate.is_file():
|
|
67
|
+
record_file = candidate
|
|
68
|
+
break
|
|
69
|
+
if record_file is None:
|
|
70
|
+
raise FileNotFoundError(f"No RECORD found in unpacked wheel at {unpacked_dir}")
|
|
71
|
+
|
|
72
|
+
record_rel = record_file.relative_to(unpacked_dir).as_posix()
|
|
73
|
+
rows: list[str] = []
|
|
74
|
+
with tempfile.NamedTemporaryFile(delete=False, suffix=".whl") as tmp:
|
|
75
|
+
tmp_path = Path(tmp.name)
|
|
76
|
+
try:
|
|
77
|
+
with zipfile.ZipFile(tmp_path, "w", zipfile.ZIP_DEFLATED) as zf:
|
|
78
|
+
for root, _, files in os.walk(unpacked_dir):
|
|
79
|
+
for name in sorted(files):
|
|
80
|
+
full = Path(root) / name
|
|
81
|
+
rel = full.relative_to(unpacked_dir).as_posix()
|
|
82
|
+
if rel == record_rel:
|
|
83
|
+
continue
|
|
84
|
+
data = full.read_bytes()
|
|
85
|
+
zf.writestr(rel, data)
|
|
86
|
+
rows.append(f"{rel},{_hash_record_entry(data)},{len(data)}")
|
|
87
|
+
rows.append(f"{record_rel},,")
|
|
88
|
+
zf.writestr(record_rel, "\n".join(rows) + "\n")
|
|
89
|
+
shutil.move(str(tmp_path), str(wheel_path))
|
|
90
|
+
finally:
|
|
91
|
+
if tmp_path.exists():
|
|
92
|
+
tmp_path.unlink()
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def _find_extensions(tree: Path, suffixes: tuple[str, ...]) -> list[Path]:
|
|
96
|
+
hits = []
|
|
97
|
+
for suffix in suffixes:
|
|
98
|
+
hits.extend(tree.rglob(f"_lbug*{suffix}"))
|
|
99
|
+
# Skip files that live inside symbol bundles/dirs themselves. NOTE: a
|
|
100
|
+
# dSYM bundle dir is named "<binary>.dSYM", so ".dSYM" is a suffix of
|
|
101
|
+
# a path part, not a standalone part -- check endswith, not membership.
|
|
102
|
+
return [
|
|
103
|
+
p
|
|
104
|
+
for p in hits
|
|
105
|
+
if p.is_file()
|
|
106
|
+
and not any(part.endswith(".dSYM") for part in p.parts)
|
|
107
|
+
and not p.name.endswith(".debug")
|
|
108
|
+
]
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def _mach_o_has_dwarf(shared_obj: Path) -> bool:
|
|
112
|
+
"""Best-effort check for a __DWARF segment (False if unknown is unsafe).
|
|
113
|
+
Returns True when unsure so callers attempt a split rather than reuse."""
|
|
114
|
+
otool = shutil.which("otool")
|
|
115
|
+
if otool is None:
|
|
116
|
+
return True
|
|
117
|
+
try:
|
|
118
|
+
out = subprocess.run(
|
|
119
|
+
[otool, "-l", str(shared_obj)],
|
|
120
|
+
capture_output=True,
|
|
121
|
+
text=True,
|
|
122
|
+
check=False,
|
|
123
|
+
)
|
|
124
|
+
return "__DWARF" in out.stdout
|
|
125
|
+
except OSError:
|
|
126
|
+
return True
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def _split_linux(wheel_path: Path, tree: Path, symbols_dir: Path) -> None:
|
|
130
|
+
objcopy = shutil.which("objcopy")
|
|
131
|
+
if objcopy is None:
|
|
132
|
+
print("objcopy not found; leaving Linux wheel unstripped.", flush=True)
|
|
133
|
+
return
|
|
134
|
+
for so in _find_extensions(tree, (".so",)):
|
|
135
|
+
stem = f"{wheel_path.stem}-{so.name}"
|
|
136
|
+
shipped = list(tree.rglob("*.debug"))
|
|
137
|
+
if shipped and not _has_debug_sections(so):
|
|
138
|
+
# In-build CMake split already stripped this binary; reuse it.
|
|
139
|
+
for dbg in shipped:
|
|
140
|
+
dest = symbols_dir / f"{stem}.debug"
|
|
141
|
+
print(f"Reusing shipped debug file {dbg} -> {dest}", flush=True)
|
|
142
|
+
shutil.move(str(dbg), str(dest))
|
|
143
|
+
continue
|
|
144
|
+
# Regenerate from the final (repaired) binary.
|
|
145
|
+
for dbg in shipped:
|
|
146
|
+
dbg.unlink()
|
|
147
|
+
dbg_out = so.parent / f"{so.name}.debug"
|
|
148
|
+
_run([objcopy, "--only-keep-debug", str(so), str(dbg_out)])
|
|
149
|
+
_run([objcopy, "--strip-debug", str(so)])
|
|
150
|
+
# --strip-debug leaves a pre-existing .gnu_debuglink behind, and
|
|
151
|
+
# --add-gnu-debuglink refuses to overwrite it, so drop it first.
|
|
152
|
+
subprocess.run(
|
|
153
|
+
[objcopy, "-R", ".gnu_debuglink", str(so)],
|
|
154
|
+
capture_output=True,
|
|
155
|
+
check=False,
|
|
156
|
+
)
|
|
157
|
+
# NOTE: --add-gnu-debuglink resolves a relative filename against the
|
|
158
|
+
# process CWD (not the binary's dir), so run from so.parent so the
|
|
159
|
+
# stored link stays a bare filename while objcopy can read the file.
|
|
160
|
+
_run([objcopy, f"--add-gnu-debuglink={dbg_out.name}", str(so)], cwd=so.parent)
|
|
161
|
+
dest = symbols_dir / f"{stem}.debug"
|
|
162
|
+
print(f"Moving {dbg_out} -> {dest}", flush=True)
|
|
163
|
+
shutil.move(str(dbg_out), str(dest))
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def _split_macos(wheel_path: Path, tree: Path, symbols_dir: Path) -> None:
|
|
167
|
+
dsymutil = shutil.which("dsymutil")
|
|
168
|
+
strip = shutil.which("strip")
|
|
169
|
+
if dsymutil is None or strip is None:
|
|
170
|
+
print("dsymutil/strip not found; leaving macOS wheel unstripped.", flush=True)
|
|
171
|
+
return
|
|
172
|
+
for so in _find_extensions(tree, (".so",)):
|
|
173
|
+
stem = f"{wheel_path.stem}-{so.name}"
|
|
174
|
+
shipped = [d for d in tree.rglob("*.dSYM")]
|
|
175
|
+
if shipped and not _mach_o_has_dwarf(so):
|
|
176
|
+
# In-build split already stripped this binary; reuse its bundle.
|
|
177
|
+
for dsym in shipped:
|
|
178
|
+
dest = symbols_dir / f"{stem}.dSYM"
|
|
179
|
+
if dest.exists():
|
|
180
|
+
shutil.rmtree(dest) if dest.is_dir() else dest.unlink()
|
|
181
|
+
print(f"Reusing shipped dSYM {dsym} -> {dest}", flush=True)
|
|
182
|
+
shutil.move(str(dsym), str(dest))
|
|
183
|
+
continue
|
|
184
|
+
# Otherwise drop any stale pre-repair bundle and regenerate below so
|
|
185
|
+
# the dSYM matches the final delocated binary.
|
|
186
|
+
for stale in shipped:
|
|
187
|
+
if stale.is_dir():
|
|
188
|
+
shutil.rmtree(stale)
|
|
189
|
+
elif stale.is_file():
|
|
190
|
+
stale.unlink()
|
|
191
|
+
dsym_out = Path(tempfile.mkdtemp()) / f"{so.name}.dSYM"
|
|
192
|
+
_run([dsymutil, str(so), "-o", str(dsym_out)])
|
|
193
|
+
_run([strip, "-S", str(so)])
|
|
194
|
+
dest = symbols_dir / f"{stem}.dSYM"
|
|
195
|
+
if dest.exists():
|
|
196
|
+
shutil.rmtree(dest)
|
|
197
|
+
print(f"Moving {dsym_out} -> {dest}", flush=True)
|
|
198
|
+
shutil.move(str(dsym_out), str(dest))
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
def _split_windows(wheel_path: Path, tree: Path, symbols_dir: Path) -> None:
|
|
202
|
+
stem = wheel_path.stem
|
|
203
|
+
for pattern in ("*.pdb", "*.exp", "*.lib"):
|
|
204
|
+
for artifact in tree.rglob(pattern):
|
|
205
|
+
if not artifact.is_file():
|
|
206
|
+
continue
|
|
207
|
+
if pattern == "*.pdb":
|
|
208
|
+
dest = symbols_dir / f"{stem}-{artifact.name}"
|
|
209
|
+
print(f"Moving {artifact} -> {dest}", flush=True)
|
|
210
|
+
shutil.move(str(artifact), str(dest))
|
|
211
|
+
else:
|
|
212
|
+
# Linker byproducts: never ship in the wheel.
|
|
213
|
+
print(f"Removing linker byproduct {artifact}", flush=True)
|
|
214
|
+
artifact.unlink()
|
|
215
|
+
# The .pyd linked with /DEBUG already carries no debug data; nothing to strip.
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
def process_wheel(wheel_path: Path, symbols_dir: Path, os_name: str) -> None:
|
|
219
|
+
print(f"=== {wheel_path.name} ({os_name}) ===", flush=True)
|
|
220
|
+
with tempfile.TemporaryDirectory() as tmp:
|
|
221
|
+
tree = Path(tmp) / "wheel"
|
|
222
|
+
with zipfile.ZipFile(wheel_path, "r") as zf:
|
|
223
|
+
zf.extractall(tree)
|
|
224
|
+
if os_name == "linux":
|
|
225
|
+
_split_linux(wheel_path, tree, symbols_dir)
|
|
226
|
+
elif os_name == "macos":
|
|
227
|
+
_split_macos(wheel_path, tree, symbols_dir)
|
|
228
|
+
elif os_name == "windows":
|
|
229
|
+
_split_windows(wheel_path, tree, symbols_dir)
|
|
230
|
+
else:
|
|
231
|
+
raise ValueError(f"Unknown os: {os_name}")
|
|
232
|
+
_repack_wheel(tree, wheel_path)
|
|
233
|
+
print(f"Repacked stripped wheel: {wheel_path.name}", flush=True)
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
def main() -> int:
|
|
237
|
+
parser = argparse.ArgumentParser(description=__doc__)
|
|
238
|
+
parser.add_argument("--wheel-dir", required=True, help="Directory with *.whl files")
|
|
239
|
+
parser.add_argument("--symbols-dir", required=True, help="Output dir for symbols")
|
|
240
|
+
parser.add_argument("--os", required=True, choices=["linux", "macos", "windows"])
|
|
241
|
+
args = parser.parse_args()
|
|
242
|
+
|
|
243
|
+
wheel_dir = Path(args.wheel_dir)
|
|
244
|
+
symbols_dir = Path(args.symbols_dir)
|
|
245
|
+
symbols_dir.mkdir(parents=True, exist_ok=True)
|
|
246
|
+
|
|
247
|
+
wheels = sorted(wheel_dir.glob("*.whl"))
|
|
248
|
+
if not wheels:
|
|
249
|
+
print(f"No wheels found in {wheel_dir}; nothing to do.", flush=True)
|
|
250
|
+
return 0
|
|
251
|
+
for wheel in wheels:
|
|
252
|
+
process_wheel(wheel, symbols_dir, args.os)
|
|
253
|
+
|
|
254
|
+
produced = sorted(p.name for p in symbols_dir.iterdir())
|
|
255
|
+
print(f"Symbol files produced ({len(produced)}):", flush=True)
|
|
256
|
+
for name in produced:
|
|
257
|
+
print(f" {name}", flush=True)
|
|
258
|
+
return 0
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
if __name__ == "__main__":
|
|
262
|
+
sys.exit(main())
|
|
@@ -57,7 +57,7 @@ iC_ExportDatabase
|
|
|
57
57
|
: EXPORT SP DATABASE SP StringLiteral ( SP? '(' SP? iC_Options SP? ')' )? ;
|
|
58
58
|
|
|
59
59
|
iC_ImportDatabase
|
|
60
|
-
: IMPORT SP DATABASE SP StringLiteral;
|
|
60
|
+
: IMPORT SP DATABASE SP StringLiteral ( SP? '(' SP? iC_Options SP? ')' )? ;
|
|
61
61
|
|
|
62
62
|
iC_AttachDatabase
|
|
63
63
|
: ATTACH SP StringLiteral (SP AS SP oC_SchemaName)? SP '(' SP? DBTYPE SP oC_SymbolicName (SP? ',' SP? iC_Options)? SP? ')' ;
|
|
@@ -11,6 +11,7 @@
|
|
|
11
11
|
#include "parser/parser.h"
|
|
12
12
|
#include "parser/port_db.h"
|
|
13
13
|
#include "parser/query/regular_query.h"
|
|
14
|
+
#include "storage/table/ice_disk_constants.h"
|
|
14
15
|
#include "transaction/transaction.h"
|
|
15
16
|
#include <format>
|
|
16
17
|
|
|
@@ -35,6 +36,10 @@ FileTypeInfo getFileType(case_insensitive_map_t<Value>& options) {
|
|
|
35
36
|
auto valueStr = value.getValue<std::string>();
|
|
36
37
|
StringUtils::toUpper(valueStr);
|
|
37
38
|
fileTypeInfo = FileTypeInfo{FileTypeUtils::fromString(valueStr), valueStr};
|
|
39
|
+
if (fileTypeInfo.fileType != FileType::PARQUET) {
|
|
40
|
+
throw BinderException(
|
|
41
|
+
"Export database only supports the parquet (icebug-disk CSR) format.");
|
|
42
|
+
}
|
|
38
43
|
options.erase(PortDBConstants::EXPORT_FORMAT_OPTION);
|
|
39
44
|
}
|
|
40
45
|
return fileTypeInfo;
|
|
@@ -57,66 +62,134 @@ void bindExportTableData(ExportedTableData& tableData, const std::string& query,
|
|
|
57
62
|
tableData.regularQuery = std::move(boundQuery);
|
|
58
63
|
}
|
|
59
64
|
|
|
65
|
+
static std::string getPropertyReturnList(const TableCatalogEntry& entry, const std::string& var) {
|
|
66
|
+
std::string props;
|
|
67
|
+
for (auto& property : entry.getProperties()) {
|
|
68
|
+
if (property.getType() == LogicalType::INTERNAL_ID()) {
|
|
69
|
+
continue;
|
|
70
|
+
}
|
|
71
|
+
props += std::format(",{}.{} as {}", var, StringUtils::quoteIdentifier(property.getName()),
|
|
72
|
+
StringUtils::quoteIdentifier(property.getName()));
|
|
73
|
+
}
|
|
74
|
+
return props;
|
|
75
|
+
}
|
|
76
|
+
|
|
60
77
|
static std::string getExportNodeTableDataQuery(const TableCatalogEntry& entry) {
|
|
61
|
-
|
|
78
|
+
// Order by storage offset so that row i in nodes_<table>.parquet holds the node
|
|
79
|
+
// with dense offset i, matching the CSR target/ptr values in the rel tables.
|
|
80
|
+
// Properties are aliased explicitly so parquet columns carry clean names.
|
|
81
|
+
auto props = getPropertyReturnList(entry, "a");
|
|
82
|
+
auto returns = props.empty() ? "a.*" : props.substr(1);
|
|
83
|
+
return std::format("match (a:{}) return {} order by offset(id(a))",
|
|
84
|
+
StringUtils::quoteIdentifier(entry.getName()), returns);
|
|
62
85
|
}
|
|
63
86
|
|
|
64
|
-
static std::string
|
|
87
|
+
static std::string getExportRelIndicesQuery(const TableCatalogEntry& relGroupEntry,
|
|
65
88
|
const NodeTableCatalogEntry& srcEntry, const NodeTableCatalogEntry& dstEntry) {
|
|
66
|
-
|
|
89
|
+
// CSR indices table: dense target offsets (+ edge properties), ordered by
|
|
90
|
+
// (source offset, target offset). The source offset is encoded by indptr.
|
|
91
|
+
return std::format("match (a:{})-[r:{}]->(b:{}) return offset(id(b)) as target{} "
|
|
92
|
+
"order by offset(id(a)), offset(id(b));",
|
|
67
93
|
StringUtils::quoteIdentifier(srcEntry.getName()),
|
|
68
94
|
StringUtils::quoteIdentifier(relGroupEntry.getName()),
|
|
69
95
|
StringUtils::quoteIdentifier(dstEntry.getName()),
|
|
70
|
-
|
|
71
|
-
|
|
96
|
+
getPropertyReturnList(relGroupEntry, "r"));
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
static std::string getExportRelIndptrQuery(const TableCatalogEntry& relGroupEntry,
|
|
100
|
+
const NodeTableCatalogEntry& srcEntry, const NodeTableCatalogEntry& dstEntry) {
|
|
101
|
+
// One row per source node (OPTIONAL MATCH keeps zero-degree nodes), consumed by
|
|
102
|
+
// the indptr export function which sorts by src and prefix-sums the degrees.
|
|
103
|
+
return std::format(
|
|
104
|
+
"match (a:{}) optional match (a)-[r:{}]->(b:{}) return offset(id(a)) as src, "
|
|
105
|
+
"count(r) as degree;",
|
|
106
|
+
StringUtils::quoteIdentifier(srcEntry.getName()),
|
|
107
|
+
StringUtils::quoteIdentifier(relGroupEntry.getName()),
|
|
108
|
+
StringUtils::quoteIdentifier(dstEntry.getName()));
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
static std::string getExportRelFlatQuery(const TableCatalogEntry& relGroupEntry,
|
|
112
|
+
const NodeTableCatalogEntry& srcEntry, const NodeTableCatalogEntry& dstEntry) {
|
|
113
|
+
// Flat relationship file for native re-import: source/target primary-key values
|
|
114
|
+
// (COPY FROM resolves them back to node offsets) followed by edge properties.
|
|
115
|
+
return std::format("match (a:{})-[r:{}]->(b:{}) return {}.{} as {}, {}.{} as {}{};",
|
|
116
|
+
StringUtils::quoteIdentifier(srcEntry.getName()),
|
|
117
|
+
StringUtils::quoteIdentifier(relGroupEntry.getName()),
|
|
118
|
+
StringUtils::quoteIdentifier(dstEntry.getName()), "a",
|
|
119
|
+
StringUtils::quoteIdentifier(srcEntry.getPrimaryKeyDefinition().getName()),
|
|
120
|
+
StringUtils::quoteIdentifier("from"), "b",
|
|
121
|
+
StringUtils::quoteIdentifier(dstEntry.getPrimaryKeyDefinition().getName()),
|
|
122
|
+
StringUtils::quoteIdentifier("to"), getPropertyReturnList(relGroupEntry, "r"));
|
|
72
123
|
}
|
|
73
124
|
|
|
74
125
|
static std::vector<ExportedTableData> getExportInfo(const Catalog& catalog,
|
|
75
|
-
main::ClientContext* context, Binder* binder, FileTypeInfo& fileTypeInfo) {
|
|
126
|
+
main::ClientContext* context, Binder* binder, [[maybe_unused]] FileTypeInfo& fileTypeInfo) {
|
|
76
127
|
auto transaction = Transaction::Get(*context);
|
|
77
128
|
std::vector<ExportedTableData> exportData;
|
|
78
129
|
for (auto entry : catalog.getNodeTableEntries(transaction, false /*useInternal*/)) {
|
|
79
130
|
ExportedTableData tableData;
|
|
80
131
|
tableData.tableName = entry->getName();
|
|
81
|
-
tableData.fileName =
|
|
82
|
-
entry->getName() + "." + StringUtils::getLower(fileTypeInfo.fileTypeStr);
|
|
132
|
+
tableData.fileName = std::format("nodes_{}.parquet", entry->getName());
|
|
83
133
|
auto query = getExportNodeTableDataQuery(*entry);
|
|
84
134
|
bindExportTableData(tableData, query, context, binder);
|
|
85
135
|
exportData.push_back(std::move(tableData));
|
|
86
136
|
}
|
|
87
137
|
for (auto entry : catalog.getRelGroupEntries(transaction, false /* useInternal */)) {
|
|
88
138
|
auto& relGroupEntry = entry->constCast<RelGroupCatalogEntry>();
|
|
139
|
+
if (relGroupEntry.getRelEntryInfos().size() != 1) {
|
|
140
|
+
throw BinderException(std::format(
|
|
141
|
+
"Export database: rel table {} connects multiple node pairs, which is not "
|
|
142
|
+
"supported for icebug-disk CSR export yet.",
|
|
143
|
+
relGroupEntry.getName()));
|
|
144
|
+
}
|
|
89
145
|
for (auto& info : relGroupEntry.getRelEntryInfos()) {
|
|
90
|
-
ExportedTableData tableData;
|
|
91
146
|
auto srcTableID = info.nodePair.srcTableID;
|
|
92
147
|
auto dstTableID = info.nodePair.dstTableID;
|
|
93
148
|
auto& srcEntry = catalog.getTableCatalogEntry(transaction, srcTableID)
|
|
94
149
|
->constCast<NodeTableCatalogEntry>();
|
|
95
150
|
auto& dstEntry = catalog.getTableCatalogEntry(transaction, dstTableID)
|
|
96
151
|
->constCast<NodeTableCatalogEntry>();
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
152
|
+
ExportedTableData indicesData;
|
|
153
|
+
indicesData.tableName = entry->getName();
|
|
154
|
+
indicesData.fileName = std::format("indices_{}.parquet", relGroupEntry.getName());
|
|
155
|
+
bindExportTableData(indicesData,
|
|
156
|
+
getExportRelIndicesQuery(relGroupEntry, srcEntry, dstEntry), context, binder);
|
|
157
|
+
exportData.push_back(std::move(indicesData));
|
|
158
|
+
ExportedTableData indptrData;
|
|
159
|
+
indptrData.tableName = entry->getName();
|
|
160
|
+
indptrData.fileName = std::format("indptr_{}.parquet", relGroupEntry.getName());
|
|
161
|
+
indptrData.isIndptr = true;
|
|
162
|
+
bindExportTableData(indptrData,
|
|
163
|
+
getExportRelIndptrQuery(relGroupEntry, srcEntry, dstEntry), context, binder);
|
|
164
|
+
exportData.push_back(std::move(indptrData));
|
|
165
|
+
// Flat relationship file (from/to primary-key values + properties), used by
|
|
166
|
+
// IMPORT DATABASE to ingest the relationship table into native storage. It is
|
|
167
|
+
// ignored when the export is mounted as icebug-disk (which uses the CSR files).
|
|
168
|
+
ExportedTableData flatRelData;
|
|
169
|
+
flatRelData.tableName = entry->getName();
|
|
170
|
+
flatRelData.fileName = std::format("rels_{}.parquet", relGroupEntry.getName());
|
|
171
|
+
bindExportTableData(flatRelData,
|
|
172
|
+
getExportRelFlatQuery(relGroupEntry, srcEntry, dstEntry), context, binder);
|
|
173
|
+
exportData.push_back(std::move(flatRelData));
|
|
104
174
|
}
|
|
105
175
|
}
|
|
106
176
|
|
|
177
|
+
// Index aux tables (e.g. the FTS internal stopwords table) are not part of the CSR
|
|
178
|
+
// layout, but index.cypher needs them on import when the original stopwords source
|
|
179
|
+
// table no longer exists. Export them as <tableName>.parquet; the FTS index's
|
|
180
|
+
// toCypher references the file path in that case.
|
|
107
181
|
for (auto indexEntry : catalog.getIndexEntries(transaction)) {
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
auto entry = indexEntry->getTableEntryToExport(context);
|
|
111
|
-
if (entry == nullptr) {
|
|
182
|
+
auto auxEntry = indexEntry->getTableEntryToExport(context);
|
|
183
|
+
if (auxEntry == nullptr) {
|
|
112
184
|
continue;
|
|
113
185
|
}
|
|
114
|
-
|
|
115
|
-
tableData.tableName =
|
|
116
|
-
tableData.fileName =
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
186
|
+
ExportedTableData tableData;
|
|
187
|
+
tableData.tableName = auxEntry->getName();
|
|
188
|
+
tableData.fileName = std::format("{}.parquet", auxEntry->getName());
|
|
189
|
+
bindExportTableData(tableData,
|
|
190
|
+
std::format("match (a:{}) return a.*",
|
|
191
|
+
StringUtils::quoteIdentifier(auxEntry->getName())),
|
|
192
|
+
context, binder);
|
|
120
193
|
exportData.push_back(std::move(tableData));
|
|
121
194
|
}
|
|
122
195
|
return exportData;
|
|
@@ -158,10 +231,17 @@ std::unique_ptr<BoundStatement> Binder::bindExportDatabaseClause(const Statement
|
|
|
158
231
|
}
|
|
159
232
|
auto exportSchemaOnly = schemaOnly(parsedOptions, exportDB);
|
|
160
233
|
if (!exportSchemaOnly && fileTypeInfo.fileType != FileType::CSV && parsedOptions.size() != 0) {
|
|
161
|
-
throw BinderException{"
|
|
234
|
+
throw BinderException{"Export database does not support copy options with the parquet "
|
|
235
|
+
"(icebug-disk CSR) format."};
|
|
162
236
|
}
|
|
163
237
|
auto exportData =
|
|
164
238
|
getExportInfo(*Catalog::Get(*clientContext), clientContext, this, fileTypeInfo);
|
|
239
|
+
// Stamp every exported parquet file with the icebug-disk format version so that
|
|
240
|
+
// IceDiskUtils::checkVersionCompatibility can validate it when the export is mounted.
|
|
241
|
+
if (fileTypeInfo.fileType == FileType::PARQUET && !exportSchemaOnly) {
|
|
242
|
+
parsedOptions.insert_or_assign(PortDBConstants::ICEBUG_DISK_VERSION_OPTION,
|
|
243
|
+
Value{std::string{storage::IceDiskConstants::CURRENT_VERSION}});
|
|
244
|
+
}
|
|
165
245
|
auto boundFilePath = VirtualFileSystem::GetUnsafe(*clientContext)
|
|
166
246
|
->expandPath(clientContext, exportDB.getFilePath());
|
|
167
247
|
return std::make_unique<BoundExportDatabase>(boundFilePath, fileTypeInfo, std::move(exportData),
|