chdb 3.6.0__cp38-abi3-macosx_11_0_arm64.whl → 3.7.0__cp38-abi3-macosx_11_0_arm64.whl

chdb/build.sh

@@ -0,0 +1,370 @@
+#!/bin/bash
+
+set -e
+
+# default to build Release
+build_type=${1:-Release}
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
+. ${DIR}/vars.sh
+
+BUILD_DIR=${PROJ_DIR}/buildlib
+
+HDFS="-DENABLE_HDFS=1 -DENABLE_GSASL_LIBRARY=1 -DENABLE_KRB5=1"
+MYSQL="-DENABLE_MYSQL=1"
+RUST_FEATURES="-DENABLE_RUST=0"
+# check current os type
+if [ "$(uname)" == "Darwin" ]; then
+    export CXX=$(brew --prefix llvm@19)/bin/clang++
+    export CC=$(brew --prefix llvm@19)/bin/clang
+    export PATH=$(brew --prefix llvm@19)/bin:$PATH
+    GLIBC_COMPATIBILITY="-DGLIBC_COMPATIBILITY=0"
+    UNWIND="-DUSE_UNWIND=0"
+    JEMALLOC="-DENABLE_JEMALLOC=0"
+    PYINIT_ENTRY="-Wl,-exported_symbol,_PyInit_${CHDB_PY_MOD}"
+    HDFS="-DENABLE_HDFS=0 -DENABLE_GSASL_LIBRARY=0 -DENABLE_KRB5=0"
+    MYSQL="-DENABLE_MYSQL=0"
+    ICU="-DENABLE_ICU=0"
+    SED_INPLACE="sed -i ''"
+    # if Darwin ARM64 (M1, M2), disable AVX
+    if [ "$(uname -m)" == "arm64" ]; then
+        CPU_FEATURES="-DENABLE_AVX=0 -DENABLE_AVX2=0"
+        LLVM="-DENABLE_EMBEDDED_COMPILER=0 -DENABLE_DWARF_PARSER=0"
+    else
+        LLVM="-DENABLE_EMBEDDED_COMPILER=0 -DENABLE_DWARF_PARSER=0"
+        # disable AVX on Darwin for macos11
+        if [ "$(sw_vers -productVersion | cut -d. -f1)" -le 11 ]; then
+            CPU_FEATURES="-DENABLE_AVX=0 -DENABLE_AVX2=0"
+        else
+            # for M1, M2 using x86_64 emulation, we need to disable AVX and AVX2
+            CPU_FEATURES="-DENABLE_AVX=0 -DENABLE_AVX2=0"
+            # # If target macos version is 12, we need to test if support AVX2, 
+            # # because some Mac Pro Late 2013 (MacPro6,1) support AVX but not AVX2
+            # # just test it on the github action, hope you don't using Mac Pro Late 2013.
+            # # https://everymac.com/mac-answers/macos-12-monterey-faq/macos-monterey-macos-12-compatbility-list-system-requirements.html
+            # if [ "$(sysctl -n machdep.cpu.leaf7_features | grep AVX2)" != "" ]; then
+            #     CPU_FEATURES="-DENABLE_AVX=1 -DENABLE_AVX2=1"
+            # else
+            #     CPU_FEATURES="-DENABLE_AVX=1 -DENABLE_AVX2=0"
+            # fi
+        fi
+    fi
+elif [ "$(uname)" == "Linux" ]; then
+    GLIBC_COMPATIBILITY="-DGLIBC_COMPATIBILITY=1"
+    UNWIND="-DUSE_UNWIND=1"
+    JEMALLOC="-DENABLE_JEMALLOC=1"
+    PYINIT_ENTRY="-Wl,-ePyInit_${CHDB_PY_MOD}"
+    ICU="-DENABLE_ICU=1"
+    SED_INPLACE="sed -i"
+    # only x86_64, enable AVX, enable embedded compiler
+    if [ "$(uname -m)" == "x86_64" ]; then
+        CPU_FEATURES="-DENABLE_AVX=1 -DENABLE_AVX2=0"
+        LLVM="-DENABLE_EMBEDDED_COMPILER=1 -DENABLE_DWARF_PARSER=1"
+        RUST_FEATURES="-DENABLE_RUST=1 -DENABLE_DELTA_KERNEL_RS=1"
+        CORROSION_CMAKE_FILE="${PROJ_DIR}/contrib/corrosion-cmake/CMakeLists.txt"
+        if [ -f "${CORROSION_CMAKE_FILE}" ]; then
+            if ! grep -q 'OPENSSL_NO_DEPRECATED_3_0' "${CORROSION_CMAKE_FILE}"; then
+                echo "Modifying corrosion CMakeLists.txt for Linux x86_64..."
+                ${SED_INPLACE} 's/corrosion_set_env_vars(${target_name} "RUSTFLAGS=${RUSTFLAGS}")/corrosion_set_env_vars(${target_name} "RUSTFLAGS=${RUSTFLAGS} --cfg osslconf=\\\"OPENSSL_NO_DEPRECATED_3_0\\\"")/g' "${CORROSION_CMAKE_FILE}"
+            else
+                echo "corrosion CMakeLists.txt already modified, skipping..."
+            fi
+        else
+            echo "Warning: corrosion CMakeLists.txt not found at ${CORROSION_CMAKE_FILE}"
+        fi
+    else
+        CPU_FEATURES="-DENABLE_AVX=0 -DENABLE_AVX2=0 -DNO_ARMV81_OR_HIGHER=1"
+        LLVM="-DENABLE_EMBEDDED_COMPILER=0 -DENABLE_DWARF_PARSER=0"
+    fi
+else
+    echo "OS not supported"
+    exit 1
+fi
+
+if [ ! -d $BUILD_DIR ]; then
+    mkdir $BUILD_DIR
+fi
+
+cd ${BUILD_DIR}
+CMAKE_ARGS="-DCMAKE_BUILD_TYPE=${build_type} -DENABLE_THINLTO=0 -DENABLE_TESTS=0 -DENABLE_CLICKHOUSE_SERVER=0 -DENABLE_CLICKHOUSE_CLIENT=0 \
+    -DENABLE_CLICKHOUSE_KEEPER=0 -DENABLE_CLICKHOUSE_KEEPER_CONVERTER=0 -DENABLE_CLICKHOUSE_LOCAL=1 -DENABLE_CLICKHOUSE_SU=0 -DENABLE_CLICKHOUSE_BENCHMARK=0 \
+    -DENABLE_AZURE_BLOB_STORAGE=1 -DENABLE_CLICKHOUSE_COPIER=0 -DENABLE_CLICKHOUSE_DISKS=0 -DENABLE_CLICKHOUSE_FORMAT=0 -DENABLE_CLICKHOUSE_GIT_IMPORT=0 \
+    -DENABLE_AWS_S3=1 -DENABLE_HIVE=0 -DENABLE_AVRO=1 \
+    -DENABLE_CLICKHOUSE_OBFUSCATOR=0 -DENABLE_CLICKHOUSE_ODBC_BRIDGE=0 -DENABLE_CLICKHOUSE_STATIC_FILES_DISK_UPLOADER=0 \
+    -DENABLE_KAFKA=1 -DENABLE_LIBPQXX=1 -DENABLE_NATS=0 -DENABLE_AMQPCPP=0 -DENABLE_NURAFT=0 \
+    -DENABLE_CASSANDRA=0 -DENABLE_ODBC=0 -DENABLE_NLP=0 \
+    -DENABLE_LDAP=0 \
+    ${MYSQL} \
+    ${HDFS} \
+    -DENABLE_LIBRARIES=0 ${RUST_FEATURES} \
+    ${GLIBC_COMPATIBILITY} \
+    -DENABLE_UTILS=0 ${LLVM} ${UNWIND} \
+    ${ICU} -DENABLE_UTF8PROC=1 ${JEMALLOC} \
+    -DENABLE_PARQUET=1 -DENABLE_ROCKSDB=1 -DENABLE_SQLITE=1 -DENABLE_VECTORSCAN=1 \
+    -DENABLE_PROTOBUF=1 -DENABLE_THRIFT=1 -DENABLE_MSGPACK=1 \
+    -DENABLE_BROTLI=1 -DENABLE_H3=1 -DENABLE_CURL=1 \
+    -DENABLE_CLICKHOUSE_ALL=0 -DUSE_STATIC_LIBRARIES=1 -DSPLIT_SHARED_LIBRARIES=0 \
+    -DENABLE_SIMDJSON=1 -DENABLE_RAPIDJSON=1 \
+    ${CPU_FEATURES} \
+    -DENABLE_AVX512=0 -DENABLE_AVX512_VBMI=0 \
+    -DENABLE_LIBFIU=1 \
+    -DCHDB_VERSION=${CHDB_VERSION} \
+    "
+
+# # Generate libchdb.so linkage command:
+# #   1. Use ar to delete the LocalChdb.cpp.o from libclickhouse-local-lib.a
+# #       `ar d programs/local/libclickhouse-local-lib.a LocalChdb.cpp.o`
+# #   2. Change the entry point from `PyInit_chdb` to `query_stable`
+# #       `-Wl,-ePyInit_chdb` to `-Wl,-equery_stable` on Linux
+# #       `-Wl,-exported_symbol,_PyInit_${CHDB_PY_MOD}` to 
+# #           `-Wl,-exported_symbol,_query_stable -Wl,-exported_symbol,_free_result` on Darwin
+# #   3. Change the output file name from `_chdb.cpython-xx-x86_64-linux-gnu.s` to `libchdb.so`
+# #       `-o _chdb.cpython-39-x86_64-linux-gnu.so` to `-o libchdb.so`
+# #   4. Write the command to a file for debug
+# #   5. Run the command to generate libchdb.so
+
+# # Remove object from archive and save it to a new archive like:
+# # path/to/oldname.a -> path/to/oldname-nopy.a
+# remove_obj_from_archive() {
+#     local archive=$1
+#     local obj=$2
+#     local new_archive=$(echo ${archive} | sed 's/\.a$/-nopy.a/')
+#     cp -a ${archive} ${new_archive}
+#     ${AR} d ${new_archive} ${obj}
+#     echo "Old archive: ${archive}"
+#     ls -l ${archive}
+#     echo "New archive: ${new_archive}"
+#     ls -l ${new_archive}
+#     local oldfile=$(basename ${archive})
+#     local newfile=$(basename ${new_archive})
+#     LIBCHDB_CMD=$(echo ${LIBCHDB_CMD} | sed "s/${oldfile}/${newfile}/g")
+#     ${SED_INPLACE} "s/${oldfile}/${newfile}/g" CMakeFiles/libchdb.rsp
+# }
+
+
+# # Step 1, 2, 3:
+# #   Backup the libclickhouse-local-lib.a and restore it after ar d
+# # LIBCHDB_SO="libchdb.so"
+# # CLEAN_CHDB_A="libclickhouse-local-chdb.a"
+# # cp -a ${BUILD_DIR}/programs/local/libclickhouse-local-lib.a ${BUILD_DIR}/programs/local/libclickhouse-local-lib.a.bak
+# # ${AR} d ${BUILD_DIR}/programs/local/libclickhouse-local-lib.a LocalChdb.cpp.o
+# # mv ${BUILD_DIR}/programs/local/libclickhouse-local-lib.a ${BUILD_DIR}/programs/local/${CLEAN_CHDB_A}
+# # mv ${BUILD_DIR}/programs/local/libclickhouse-local-lib.a.bak ${BUILD_DIR}/programs/local/libclickhouse-local-lib.a
+# # ls -l ${BUILD_DIR}/programs/local/
+# LIBCHDB_SO="libchdb.so"
+# LIBCHDB_CMD=${PYCHDB_CMD}
+# if [ "${build_type}" == "Debug" ]; then
+#     remove_obj_from_archive ${BUILD_DIR}/programs/local/libclickhouse-local-libd.a LocalChdb.cpp.o
+#     remove_obj_from_archive ${BUILD_DIR}/src/libdbmsd.a StoragePython.cpp.o
+#     remove_obj_from_archive ${BUILD_DIR}/src/libdbmsd.a PythonSource.cpp.o
+#     remove_obj_from_archive ${BUILD_DIR}/src/libclickhouse_common_iod.a PythonUtils.cpp.o
+#     remove_obj_from_archive ${BUILD_DIR}/src/TableFunctions/libclickhouse_table_functionsd.a TableFunctionPython.cpp.o
+# else
+#     remove_obj_from_archive ${BUILD_DIR}/programs/local/libclickhouse-local-lib.a LocalChdb.cpp.o
+#     remove_obj_from_archive ${BUILD_DIR}/src/libdbms.a StoragePython.cpp.o
+#     remove_obj_from_archive ${BUILD_DIR}/src/libdbms.a PythonSource.cpp.o
+#     remove_obj_from_archive ${BUILD_DIR}/src/libclickhouse_common_io.a PythonUtils.cpp.o
+#     remove_obj_from_archive ${BUILD_DIR}/src/TableFunctions/libclickhouse_table_functions.a TableFunctionPython.cpp.o
+# fi
+
+
+LIBCHDB_SO="libchdb.so"
+# Build libchdb.so
+cmake ${CMAKE_ARGS} -DENABLE_PYTHON=0 ..
+ninja -d keeprsp
+
+
+BINARY=${BUILD_DIR}/programs/clickhouse
+echo -e "\nBINARY: ${BINARY}"
+ls -lh ${BINARY}
+echo -e "\nldd ${BINARY}"
+${LDD} ${BINARY}
+rm -f ${BINARY}
+
+cd ${BUILD_DIR}
+ninja -d keeprsp -v > build.log || true
+USING_RESPONSE_FILE=$(grep -m 1 'clang++.*-o programs/clickhouse .*' build.log | grep '@CMakeFiles/clickhouse.rsp' || true)
+
+if [ ! "${USING_RESPONSE_FILE}" == "" ]; then
+    if [ -f CMakeFiles/clickhouse.rsp ]; then
+        cp -a CMakeFiles/clickhouse.rsp CMakeFiles/libchdb.rsp
+    else
+        echo "CMakeFiles/clickhouse.rsp not found"
+        exit 1
+    fi
+fi
+
+LIBCHDB_CMD=$(grep -m 1 'clang++.*-o programs/clickhouse .*' build.log \
+    | sed "s/-o programs\/clickhouse/-fPIC -shared -o ${LIBCHDB_SO}/" \
+    | sed 's/^[^&]*&& //' | sed 's/&&.*//' \
+    | sed 's/ -Wl,-undefined,error/ -Wl,-undefined,dynamic_lookup/g' \
+    | sed 's/ -Xlinker --no-undefined//g' \
+    | sed 's/@CMakeFiles\/clickhouse.rsp/@CMakeFiles\/libchdb.rsp/g' \
+     )
+
+#   generate the command to generate libchdb.so
+LIBCHDB_CMD=$(echo ${LIBCHDB_CMD} | sed 's/ '${CHDB_PY_MODULE}'/ '${LIBCHDB_SO}'/g')
+
+if [ ! "${USING_RESPONSE_FILE}" == "" ]; then
+    ${SED_INPLACE} 's/ '${CHDB_PY_MODULE}'/ '${LIBCHDB_SO}'/g' CMakeFiles/libchdb.rsp
+fi
+
+if [ "$(uname)" == "Linux" ]; then
+    LIBCHDB_CMD=$(echo ${LIBCHDB_CMD} | sed 's/ '${PYINIT_ENTRY}'/ /g')
+    if [ ! "${USING_RESPONSE_FILE}" == "" ]; then
+        ${SED_INPLACE} 's/ '${PYINIT_ENTRY}'/ /g' CMakeFiles/libchdb.rsp
+    fi
+fi
+
+if [ "$(uname)" == "Darwin" ]; then
+    LIBCHDB_CMD=$(echo ${LIBCHDB_CMD} | sed 's/ '${PYINIT_ENTRY}'/ -Wl,-exported_symbol,_query_stable -Wl,-exported_symbol,_free_result -Wl,-exported_symbol,_query_stable_v2 -Wl,-exported_symbol,_free_result_v2/g')
+    # ${SED_INPLACE} 's/ '${PYINIT_ENTRY}'/ -Wl,-exported_symbol,_query_stable -Wl,-exported_symbol,_free_result -Wl,-exported_symbol,_query_stable_v2 -Wl,-exported_symbol,_free_result_v2/g' CMakeFiles/libchdb.rsp
+fi
+
+LIBCHDB_CMD=$(echo ${LIBCHDB_CMD} | sed 's/@CMakeFiles\/clickhouse.rsp/@CMakeFiles\/libchdb.rsp/g')
+
+# Step 4:
+#   save the command to a file for debug
+echo ${LIBCHDB_CMD} > libchdb_cmd.sh
+
+# Step 5:
+${LIBCHDB_CMD}
+
+LIBCHDB_DIR=${BUILD_DIR}/
+LIBCHDB=${LIBCHDB_DIR}/${LIBCHDB_SO}
+ls -lh ${LIBCHDB}
+
+# build chdb python module
+py_version="3.8"
+# check current os type and architecture for py_version
+if [ "$(uname)" == "Darwin" ] && [ "$(uname -m)" == "x86_64" ]; then
+    py_version="3.9"
+fi
+current_py_version=$(python -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')")
+if [ "$current_py_version" != "$py_version" ]; then
+    echo "Error: Current Python version is $current_py_version, but required version is $py_version"
+    echo "Please switch to Python $py_version using: pyenv shell $py_version"
+    exit 1
+fi
+echo "Using Python version: $current_py_version"
+cmake ${CMAKE_ARGS} -DENABLE_PYTHON=1 -DPYBIND11_NONLIMITEDAPI_PYTHON_HEADERS_VERSION=${py_version} ..
+ninja -d keeprsp || true
+
+# del the binary and run ninja -v again to capture the command, then modify it to generate CHDB_PY_MODULE
+/bin/rm -f ${BINARY}
+cd ${BUILD_DIR}
+ninja -d keeprsp -v > build.log || true
+
+USING_RESPONSE_FILE=$(grep -m 1 'clang++.*-o programs/clickhouse .*' build.log | grep '@CMakeFiles/clickhouse.rsp' || true)
+
+if [ ! "${USING_RESPONSE_FILE}" == "" ]; then
+    if [ -f CMakeFiles/clickhouse.rsp ]; then
+        cp -a CMakeFiles/clickhouse.rsp CMakeFiles/pychdb.rsp
+    else
+        echo "CMakeFiles/clickhouse.rsp not found"
+        exit 1
+    fi
+fi
+
+# extract the command to generate CHDB_PY_MODULE
+PYCHDB_CMD=$(grep -m 1 'clang++.*-o programs/clickhouse .*' build.log \
+    | sed "s/-o programs\/clickhouse/-fPIC -Wl,-undefined,dynamic_lookup -shared ${PYINIT_ENTRY} -o ${CHDB_PY_MODULE}/" \
+    | sed 's/^[^&]*&& //' | sed 's/&&.*//' \
+    | sed 's/ -Wl,-undefined,error/ -Wl,-undefined,dynamic_lookup/g' \
+    | sed 's/ -Xlinker --no-undefined//g' \
+    | sed 's/@CMakeFiles\/clickhouse.rsp/@CMakeFiles\/pychdb.rsp/g' \
+     )
+
+
+# # inplace modify the CMakeFiles/pychdb.rsp
+# ${SED_INPLACE} 's/-o programs\/clickhouse/-fPIC -Wl,-undefined,dynamic_lookup -shared ${PYINIT_ENTRY} -o ${CHDB_PY_MODULE}/' CMakeFiles/pychdb.rsp
+# ${SED_INPLACE} 's/ -Wl,-undefined,error/ -Wl,-undefined,dynamic_lookup/g' CMakeFiles/pychdb.rsp
+# ${SED_INPLACE} 's/ -Xlinker --no-undefined//g' CMakeFiles/pychdb.rsp
+
+
+if [ "$(uname)" == "Linux" ]; then
+    # remove src/CMakeFiles/clickhouse_malloc.dir/Common/stubFree.c.o
+    PYCHDB_CMD=$(echo ${PYCHDB_CMD} | sed 's/ src\/CMakeFiles\/clickhouse_malloc.dir\/Common\/stubFree.c.o//g')
+    # put -Wl,-wrap,malloc ... after -DUSE_JEMALLOC=1
+    PYCHDB_CMD=$(echo ${PYCHDB_CMD} | sed 's/ -DUSE_JEMALLOC=1/ -DUSE_JEMALLOC=1 -Wl,-wrap,malloc -Wl,-wrap,valloc -Wl,-wrap,pvalloc -Wl,-wrap,calloc -Wl,-wrap,realloc -Wl,-wrap,memalign -Wl,-wrap,aligned_alloc -Wl,-wrap,posix_memalign -Wl,-wrap,free/g')
+    if [ ! "${USING_RESPONSE_FILE}" == "" ]; then
+        ${SED_INPLACE} 's/ src\/CMakeFiles\/clickhouse_malloc.dir\/Common\/stubFree.c.o//g' CMakeFiles/pychdb.rsp
+        ${SED_INPLACE} 's/ -DUSE_JEMALLOC=1/ -DUSE_JEMALLOC=1 -Wl,-wrap,malloc -Wl,-wrap,valloc -Wl,-wrap,pvalloc -Wl,-wrap,calloc -Wl,-wrap,realloc -Wl,-wrap,memalign -Wl,-wrap,aligned_alloc -Wl,-wrap,posix_memalign -Wl,-wrap,free/g' CMakeFiles/pychdb.rsp
+    fi
+fi
+
+if [ "$(uname)" == "Darwin" ]; then
+    PYCHDB_CMD=$(echo ${PYCHDB_CMD} | sed 's|-Wl,-rpath,/[^[:space:]]*/pybind11-cmake|-Wl,-rpath,@loader_path|g')
+else
+    PYCHDB_CMD=$(echo ${PYCHDB_CMD} | sed 's|-Wl,-rpath,/[^[:space:]]*/pybind11-cmake|-Wl,-rpath,\$ORIGIN|g')
+fi
+
+# save the command to a file for debug
+echo ${PYCHDB_CMD} > pychdb_cmd.sh
+
+${PYCHDB_CMD}
+
+ls -lh ${CHDB_PY_MODULE}
+
+## check all the so files
+LIBCHDB_DIR=${BUILD_DIR}/
+
+PYCHDB=${LIBCHDB_DIR}/${CHDB_PY_MODULE}
+LIBCHDB=${LIBCHDB_DIR}/${LIBCHDB_SO}
+
+if [ ${build_type} == "Debug" ]; then
+    echo -e "\nDebug build, skip strip"
+else
+    echo -e "\nStrip the binary:"
+    ${STRIP} --remove-section=.comment --remove-section=.note ${PYCHDB}
+    ${STRIP} --remove-section=.comment --remove-section=.note ${LIBCHDB}
+fi
+echo -e "\nStripe the binary:"
+
+echo -e "\nPYCHDB: ${PYCHDB}"
+ls -lh ${PYCHDB}
+echo -e "\nLIBCHDB: ${LIBCHDB}"
+ls -lh ${LIBCHDB}
+echo -e "\nldd ${PYCHDB}"
+${LDD} ${PYCHDB}
+echo -e "\nfile info of ${PYCHDB}"
+file ${PYCHDB}
+echo -e "\nldd ${LIBCHDB}"
+${LDD} ${LIBCHDB}
+echo -e "\nfile info of ${LIBCHDB}"
+file ${LIBCHDB}
+
+rm -f ${CHDB_DIR}/*.so
+cp -a ${PYCHDB} ${CHDB_DIR}/${CHDB_PY_MODULE}
+cp -a ${LIBCHDB} ${PROJ_DIR}/${LIBCHDB_SO}
+
+echo -e "\nSymbols:"
+echo -e "\nPyInit in PYCHDB: ${PYCHDB}"
+${NM} ${PYCHDB} | grep PyInit || true
+echo -e "\nPyInit in LIBCHDB: ${LIBCHDB}"
+${NM} ${LIBCHDB} | grep PyInit || echo "PyInit not found in ${LIBCHDB}, it's OK"
+echo -e "\nquery_stable in PYCHDB: ${PYCHDB}"
+${NM} ${PYCHDB} | grep query_stable || true
+echo -e "\nquery_stable in LIBCHDB: ${LIBCHDB}"
+${NM} ${LIBCHDB} | grep query_stable || true
+
+echo -e "\nAfter copy:"
+cd ${PROJ_DIR} && pwd
+# ls -lh ${PROJ_DIR}
+
+# strip the binary (no debug info at all)
+# strip ${CHDB_DIR}/${CHDB_PY_MODULE} || true
+
+# echo -e "\nAfter strip:"
+# echo -e "\nLIBCHDB: ${PYCHDB}"
+# ls -lh ${CHDB_DIR}
+# echo -e "\nfile info of ${PYCHDB}"
+# file ${CHDB_DIR}/${CHDB_PY_MODULE}
+
+ccache -s || true
+
+# bash ${DIR}/build_bind.sh
+# bash ${DIR}/test_smoke.sh
+
+CMAKE_ARGS="${CMAKE_ARGS}" bash ${DIR}/build_pybind11.sh --all

chdb/build_linux_arm64.sh

@@ -0,0 +1,63 @@
+#!/bin/bash -e
+
+export PYENV_ROOT="$HOME/.pyenv"
+[[ -d $PYENV_ROOT/bin ]] && export PATH="$PYENV_ROOT/bin:$PATH"
+eval "$(pyenv init -)"
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+PROJ_DIR=$(dirname ${DIR})
+
+[ -z "${PYTHON_VERSIONS}" ] && { echo "please provide PYTHON_VERSIONS env, e.g: PYTHON_VERSIONS='3.8 3.9'"; exit 1; }
+
+# echo ${PROJ_DIR}
+
+for PY_VER in ${PYTHON_VERSIONS}; do
+    cd ${PROJ_DIR}
+    pyenv local "${PY_VER}"
+    python3 --version
+    python3 -m pip install pybind11 setuptools
+    export CC=/usr/bin/clang
+    export CXX=/usr/bin/clang++
+    # Install universal2 pkg
+    python3 -VV
+    python3-config --includes
+    # if python3 -VV does not contain ${PY_VER}, then exit
+    if ! python3 -VV 2>&1 | grep -q "${PY_VER}"; then
+        echo "Error: Required version of Python (${PY_VER}) not found. Aborting."
+        exit 1
+    fi
+
+    python3 -m pip install -U pybind11 wheel build tox
+    rm -rf ./buildlib
+
+    ./chdb/build.sh
+    cd chdb && python3 -c "import _chdb; res = _chdb.query('select 1112222222,555', 'JSON'); print(res)" && cd -
+
+    ./gen_manifest.sh
+    cat ./MANIFEST.in
+
+    # try delete 
+    whl_file=$(find dist | grep 'whl$' | grep cp${PY_VER//./}-cp${PY_VER//./} || echo "notfound.whl")
+    rm -f ${whl_file} || :
+
+    python3 -m build --wheel
+
+    python3 -m wheel tags --platform-tag=manylinux_2_17_aarch64
+
+    find dist
+
+    python3 -m pip install pandas pyarrow psutil
+    find dist
+    whl_file=$(find dist | grep 'whl$' | grep cp${PY_VER//./}-cp${PY_VER//./})
+    python3 -m pip install --force-reinstall ${whl_file}
+
+    python3 -c "import chdb; res = chdb.query('select version()', 'CSV'); print(res)"
+
+    python3 -m chdb "SELECT 1, 'ab'" arrowtable
+
+    python3 -m chdb "SELECT 1, 'ab'" dataframe
+
+    bash -x ./chdb/test_smoke.sh
+
+    make test
+done

chdb/build_mac_arm64.sh

@@ -0,0 +1,121 @@
+#!/usr/bin/arch -arm64 /bin/bash
+
+set -e
+
+# check current arch
+if [ "$(uname -m)" != "arm64" ]; then
+    echo "OS not supported, run with arch -arm64 /bin/bash"
+    exit 1
+fi
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+PROJ_DIR=$(dirname ${DIR})
+
+cd ${PROJ_DIR}
+
+# Download py39 py310 py311 universal2 pkg from python.org
+if [ ! -d ${PROJ_DIR}/python_pkg ]; then
+    mkdir ${PROJ_DIR}/python_pkg
+fi
+
+# prefer /usr/local/opt/llvm@19/bin/clang++ then /usr/local/opt/llvm/bin/clang++
+if [ -f /usr/local/opt/llvm@19/bin/clang++ ]; then
+    export CXX=/usr/local/opt/llvm@19/bin/clang++
+elif [ -f /usr/local/opt/llvm/bin/clang++ ]; then
+    export CXX=/usr/local/opt/llvm/bin/clang++
+fi
+if [ -f /usr/local/opt/llvm@19/bin/clang ]; then
+    export CC=/usr/local/opt/llvm@19/bin/clang
+elif [ -f /usr/local/opt/llvm/bin/clang ]; then
+    export CC=/usr/local/opt/llvm/bin/clang
+fi
+
+# Download MacOSX11.0.sdk.tar.xz
+if [ ! -f ${PROJ_DIR}/python_pkg/MacOSX11.0.sdk.tar.xz ]; then
+    wget "https://github.com/phracker/MacOSX-SDKs/releases/download/11.3/MacOSX11.0.sdk.tar.xz" -O ${PROJ_DIR}/python_pkg/MacOSX11.0.sdk.tar.xz
+fi
+
+# Extract MacOSX11.0.sdk.tar.xz
+if [ ! -f ${PROJ_DIR}/cmake/toolchain/darwin-x86_64/Entitlements.plist ]; then
+    tar xJf ${PROJ_DIR}/python_pkg/MacOSX11.0.sdk.tar.xz -C cmake/toolchain/darwin-x86_64 --strip-components=1
+fi
+
+# Fix soft link if darwin-aarch64 not linked to darwin-x86_64
+if [ -L ${PROJ_DIR}/cmake/toolchain/darwin-aarch64 ]; then
+    dest=$(readlink ${PROJ_DIR}/cmake/toolchain/darwin-aarch64)
+fi
+
+if [ "${dest}" != "darwin-x86_64" ]; then
+    rm -f ${PROJ_DIR}/cmake/toolchain/darwin-aarch64
+    ln -sf darwin-x86_64 ${PROJ_DIR}/cmake/toolchain/darwin-aarch64
+fi
+
+
+for PY_VER in 3.9.13 3.10.11 3.11.3 3.12.9 3.13.2; do
+    if [ ! -f ${PROJ_DIR}/python_pkg/python-${PY_VER}-macos11.pkg ]; then
+        wget https://www.python.org/ftp/python/${PY_VER}/python-${PY_VER}-macos11.pkg -O ${PROJ_DIR}/python_pkg/python-${PY_VER}-macos11.pkg
+    fi
+    
+    PY_SHORT_VER=$(echo ${PY_VER} | cut -d. -f1,2)
+    # Install universal2 pkg
+    sudo installer -pkg ${PROJ_DIR}/python_pkg/python-${PY_VER}-macos11.pkg -target /
+    export PATH=/Library/Frameworks/Python.framework/Versions/${PY_SHORT_VER}/bin:/Library/Frameworks/Python.framework/Versions/${PY_SHORT_VER}/Resources/Python.app/Contents/MacOS/:$PATH
+    python3 -VV
+    python3-config --includes
+    # if python3 -VV does not contain ${PY_VER}, then exit
+    if ! python3 -VV 2>&1 | grep -q "${PY_VER}"; then
+        echo "Error: Required version of Python (${PY_VER}) not found. Aborting."
+        exit 1
+    fi
+    # if python3-config --includes does not contain ${PY_SHORT_VER}, then exit
+    if ! python3-config --includes 2>&1 | grep -q "${PY_SHORT_VER}"; then
+        echo "Error: Required version of Python (${PY_VER}) not found. Aborting."
+        exit 1
+    fi
+
+    python3 -m pip install -U pybind11 wheel build tox psutil setuptools pyarrow pandas certifi
+    rm -rf ${PROJ_DIR}/buildlib
+
+    ${PROJ_DIR}/chdb/build.sh
+    cd ${PROJ_DIR}/chdb && python3 -c "import _chdb; res = _chdb.query('select 1112222222,555', 'JSON'); print(res)"
+
+    cd ${PROJ_DIR}
+    ${PROJ_DIR}/gen_manifest.sh
+    cat ${PROJ_DIR}/MANIFEST.in
+
+    python3 -m build --wheel
+
+    python3 -m wheel tags --platform-tag=macosx_11_0_arm64 --remove dist/chdb-*-cp${PY_SHORT_VER//./}-cp${PY_SHORT_VER//./}-macosx_*_universal2.whl
+
+    python3 -m pip install --force-reinstall dist/chdb-*-cp${PY_SHORT_VER//./}-cp${PY_SHORT_VER//./}-macosx_11_0_arm64.whl
+
+    python3 -c "import chdb; res = chdb.query('select version()', 'CSV'); print(res)"
+
+    python3 -m chdb "SELECT 1, 'ab'" arrowtable
+
+    python3 -m chdb "SELECT 1, 'ab'" dataframe
+    
+    CHDB_VERSION=$(python3 -c "import chdb; print(chdb.__version__)")
+    make test
+done
+
+cp programs/local/chdb.h chdb.h
+tar -czvf macos-arm64-libchdb.tar.gz libchdb.so chdb.h
+gh release upload v${CHDB_VERSION} macos-arm64-libchdb.tar.gz --clobber
+
+
+cd ${PROJ_DIR}
+# Set environment variables for PyPI upload
+export TWINE_USERNAME=__token__
+export TWINE_PASSWORD=$(cat ~/.config/pypi.token)
+
+make pub
+
+# Upload wheel files to GitHub release
+echo "Uploading wheel files to GitHub release..."
+for wheel in dist/*.whl; do
+    if [ -f "$wheel" ]; then
+        gh release upload v${CHDB_VERSION} "$wheel" --clobber
+        echo "Uploaded $wheel"
+    fi
+done

chdb/build_pybind11.sh

@@ -0,0 +1,131 @@
+#!/bin/bash
+
+set -e
+
+build_all=false
+py_version=""
+
+for arg in "$@"; do
+    case $arg in
+        --all)
+            build_all=true
+            shift
+            ;;
+        --version=*)
+            py_version="${arg#*=}"
+            shift
+            ;;
+    esac
+done
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+. ${DIR}/vars.sh
+
+# Check if CMAKE_ARGS is passed from build.sh
+if [ -z "$CMAKE_ARGS" ]; then
+    echo "Error: CMAKE_ARGS not provided. This script should be called from build.sh."
+    exit 1
+fi
+
+build_pybind11_nonlimitedapi() {
+    cd "${BUILD_DIR}"
+
+    local py_version=$1
+    echo "Building pybind11 nonlimitedapi library for Python ${py_version}..."
+
+    local py_cmake_args="${CMAKE_ARGS} -DPYBIND11_NONLIMITEDAPI_PYTHON_HEADERS_VERSION=${py_version}"
+
+    cmake ${py_cmake_args} -DENABLE_PYTHON=1 ..
+
+    # Build only the pybind11 targets
+    ninja pybind11nonlimitedapi_chdb_${py_version} || {
+        echo "Failed to build pybind11nonlimitedapi library for Python ${py_version}"
+        return 1
+    }
+
+    # Copy the built library to output directory
+    local lib_name="pybind11nonlimitedapi_chdb_${py_version}"
+    if [ "$(uname)" == "Darwin" ]; then
+        local lib_file="lib${lib_name}.dylib"
+    else
+        local lib_file="lib${lib_name}.so"
+    fi
+
+    if [ -f "${BUILD_DIR}/contrib/pybind11-cmake/${lib_file}" ]; then
+        cp "${BUILD_DIR}/contrib/pybind11-cmake/${lib_file}" "${CHDB_DIR}/${lib_file}"
+        echo "Copied ${lib_file} to ${CHDB_DIR}/"
+        echo "Library location: $(realpath ${CHDB_DIR}/${lib_file})"
+    else
+        echo "Warning: ${lib_file} not found in ${BUILD_DIR}/contrib/pybind11-cmake/"
+        echo "Available files in contrib/pybind11-cmake/:"
+        ls -la "${BUILD_DIR}/contrib/pybind11-cmake/" || true
+    fi
+}
+
+build_all_pybind11_nonlimitedapi() {
+    local python_versions=("3.8" "3.9" "3.10" "3.11" "3.12" "3.13")
+    
+    # Skip Python 3.8 for macOS x86_64
+    if [ "$(uname)" == "Darwin" ] && [ "$(uname -m)" == "x86_64" ]; then
+        python_versions=("3.9" "3.10" "3.11" "3.12" "3.13")
+    fi
+
+    echo "Building pybind11 nonlimitedapi libraries for all Python versions..."
+
+    # Check if pyenv is available
+    if [ -z "$(command -v pyenv)" ]; then
+        echo "Error: pyenv not found. Please install pyenv first."
+        exit 1
+    fi
+
+    for version in "${python_versions[@]}"; do
+        # Use pyenv to find specific version
+        local pyenv_version=$(pyenv versions --bare | grep "^${version}\." | head -1)
+        if [ -z "$pyenv_version" ]; then
+            echo "Error: Python ${version} not found in pyenv. Please install it with: pyenv install ${version}.x"
+            exit 1
+        fi
+
+        echo "Found pyenv Python ${pyenv_version}"
+        export PYENV_VERSION=$pyenv_version
+
+        local python_include=$(python -c "import sysconfig; print(sysconfig.get_path('include'))" 2>/dev/null)
+        local active_version=$(python --version 2>&1)
+        echo "  Active Python: $active_version"
+
+        if [ -f "$python_include/Python.h" ]; then
+            echo "  Headers found at: $python_include"
+            build_pybind11_nonlimitedapi "${version}"
+        else
+            echo "Error: Python.h not found for Python ${version} at $python_include"
+            unset PYENV_VERSION
+            exit 1
+        fi
+
+        unset PYENV_VERSION
+    done
+
+    echo "Finished building pybind11 nonlimitedapi libraries"
+}
+
+copy_stubs() {
+    if [ "$(uname)" == "Darwin" ]; then
+        local lib_file="libpybind11nonlimitedapi_stubs.dylib"
+    else
+        local lib_file="libpybind11nonlimitedapi_stubs.so"
+    fi
+    if [ -f ${BUILD_DIR}/contrib/pybind11-cmake/${lib_file} ]; then
+        cp -a ${BUILD_DIR}/contrib/pybind11-cmake/${lib_file} ${CHDB_DIR}/
+    fi
+}
+
+if [ "$build_all" = true ]; then
+    copy_stubs
+    build_all_pybind11_nonlimitedapi
+elif [ -n "$py_version" ]; then
+    copy_stubs
+    build_pybind11_nonlimitedapi "$py_version"
+else
+    echo "No action specified."
+    exit 1
+fi