chdb 3.6.0__cp38-abi3-macosx_11_0_arm64.whl → 3.7.0__cp38-abi3-macosx_11_0_arm64.whl

chdb/.flake8

@@ -0,0 +1,5 @@
+[flake8]
+max-line-length = 130
+extend-ignore = E722
+exclude =
+    build/

chdb/__init__.py

@@ -4,7 +4,38 @@ import threading
 
 
 class ChdbError(Exception):
-    """Base class for exceptions in this module."""
+    """Base exception class for chDB-related errors.
+
+    This exception is raised when chDB query execution fails or encounters
+    an error. It inherits from the standard Python Exception class and
+    provides error information from the underlying ClickHouse engine.
+
+    The exception message typically contains detailed error information
+    from ClickHouse, including syntax errors, type mismatches, missing
+    tables/columns, and other query execution issues.
+
+    Attributes:
+        args: Tuple containing the error message and any additional arguments
+
+    Examples:
+        >>> try:
+        ...     result = chdb.query("SELECT * FROM non_existent_table")
+        ... except chdb.ChdbError as e:
+        ...     print(f"Query failed: {e}")
+        Query failed: Table 'non_existent_table' doesn't exist
+
+        >>> try:
+        ...     result = chdb.query("SELECT invalid_syntax FROM")
+        ... except chdb.ChdbError as e:
+        ...     print(f"Syntax error: {e}")
+        Syntax error: Syntax error near 'FROM'
+
+    Note:
+        This exception is automatically raised by chdb.query() and related
+        functions when the underlying ClickHouse engine reports an error.
+        You should catch this exception when handling potentially failing
+        queries to provide appropriate error handling in your application.
+    """
 
 
 _arrow_format = set({"dataframe", "arrowtable"})
@@ -19,7 +50,7 @@ _process_result_format_funs = {
 # UDF script path will be f"{g_udf_path}/{func_name}.py"
 g_udf_path = ""
 
-chdb_version = ('3', '6', '0')
+__version__ = "3.7.0"
 if sys.version_info[:2] >= (3, 7):
     # get the path of the current file
     current_path = os.path.dirname(os.path.abspath(__file__))
@@ -36,17 +67,32 @@ if sys.version_info[:2] >= (3, 7):
 else:
     raise NotImplementedError("Python 3.6 or lower version is not supported")
 
-try:
-    # Change here if project is renamed and does not equal the package name
-    dist_name = __name__
-    __version__ = ".".join(map(str, chdb_version))
-except:  # noqa
-    __version__ = "unknown"
+chdb_version = tuple(__version__.split('.'))
 
 
 # return pyarrow table
 def to_arrowTable(res):
-    """convert res to arrow table"""
+    """Convert query result to PyArrow Table.
+
+    Converts a chDB query result to a PyArrow Table for efficient columnar data processing.
+    Returns an empty table if the result is empty.
+
+    Args:
+        res: chDB query result object containing binary Arrow data
+
+    Returns:
+        pa.Table: PyArrow Table containing the query results
+
+    Raises:
+        ImportError: If pyarrow or pandas are not installed
+
+    Example:
+        >>> result = chdb.query("SELECT 1 as id, 'hello' as msg", "Arrow")
+        >>> table = chdb.to_arrowTable(result)
+        >>> print(table.to_pandas())
+           id    msg
+        0   1  hello
+    """
     # try import pyarrow and pandas, if failed, raise ImportError with suggestion
     try:
         import pyarrow as pa  # noqa
@@ -57,12 +103,34 @@ def to_arrowTable(res):
         raise ImportError("Failed to import pyarrow or pandas") from None
     if len(res) == 0:
         return pa.Table.from_batches([], schema=pa.schema([]))
-    return pa.RecordBatchFileReader(res.bytes()).read_all()
+
+    memview = res.get_memview()
+    return pa.RecordBatchFileReader(memview.view()).read_all()
 
 
 # return pandas dataframe
 def to_df(r):
-    """convert arrow table to Dataframe"""
+    """Convert query result to pandas DataFrame.
+
+    Converts a chDB query result to a pandas DataFrame by first converting to
+    PyArrow Table and then to pandas using multi-threading for better performance.
+
+    Args:
+        r: chDB query result object containing binary Arrow data
+
+    Returns:
+        pd.DataFrame: pandas DataFrame containing the query results
+
+    Raises:
+        ImportError: If pyarrow or pandas are not installed
+
+    Example:
+        >>> result = chdb.query("SELECT 1 as id, 'hello' as msg", "Arrow")
+        >>> df = chdb.to_df(result)
+        >>> print(df)
+           id    msg
+        0   1  hello
+    """
     t = to_arrowTable(r)
     return t.to_pandas(use_threads=True)
 
@@ -73,6 +141,59 @@ g_conn_lock = threading.Lock()
 
 # wrap _chdb functions
 def query(sql, output_format="CSV", path="", udf_path=""):
+    """Execute SQL query using chDB engine.
+
+    This is the main query function that executes SQL statements using the embedded
+    ClickHouse engine. Supports various output formats and can work with in-memory
+    or file-based databases.
+
+    Args:
+        sql (str): SQL query string to execute
+        output_format (str, optional): Output format for results. Defaults to "CSV".
+            Supported formats include:
+
+            - "CSV" - Comma-separated values
+            - "JSON" - JSON format
+            - "Arrow" - Apache Arrow format
+            - "Parquet" - Parquet format
+            - "DataFrame" - Pandas DataFrame
+            - "ArrowTable" - PyArrow Table
+            - "Debug" - Enable verbose logging
+
+        path (str, optional): Database file path. Defaults to "" (in-memory database).
+            Can be a file path or ":memory:" for in-memory database.
+        udf_path (str, optional): Path to User-Defined Functions directory. Defaults to "".
+
+    Returns:
+        Query result in the specified format:
+
+        - str: For text formats like CSV, JSON
+        - pd.DataFrame: When output_format is "DataFrame" or "dataframe"
+        - pa.Table: When output_format is "ArrowTable" or "arrowtable"
+        - chdb result object: For other formats
+
+    Raises:
+        ChdbError: If the SQL query execution fails
+        ImportError: If required dependencies are missing for DataFrame/Arrow formats
+
+    Examples:
+        >>> # Basic CSV query
+        >>> result = chdb.query("SELECT 1, 'hello'")
+        >>> print(result)
+        "1,hello"
+
+        >>> # Query with DataFrame output
+        >>> df = chdb.query("SELECT 1 as id, 'hello' as msg", "DataFrame")
+        >>> print(df)
+           id    msg
+        0   1  hello
+
+        >>> # Query with file-based database
+        >>> result = chdb.query("CREATE TABLE test (id INT)", path="mydb.chdb")
+
+        >>> # Query with UDF
+        >>> result = chdb.query("SELECT my_udf('test')", udf_path="/path/to/udfs")
+    """
     global g_udf_path
     if udf_path != "":
         g_udf_path = udf_path

chdb/build-musl.sh

@@ -0,0 +1,166 @@
+#!/bin/bash
+
+set -e
+
+export USE_MUSL=1
+
+build_type=${1:-Release}
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+
+. ${DIR}/vars.sh
+
+BUILD_DIR=${PROJ_DIR}/buildlib
+
+HDFS="-DENABLE_HDFS=1 -DENABLE_GSASL_LIBRARY=1 -DENABLE_KRB5=1"
+MYSQL="-DENABLE_MYSQL=1"
+RUST_FEATURES="-DENABLE_RUST=0"
+if [ "$(uname)" == "Linux" ]; then
+    GLIBC_COMPATIBILITY="-DGLIBC_COMPATIBILITY=0"
+    UNWIND="-DUSE_UNWIND=1"
+    JEMALLOC="-DENABLE_JEMALLOC=0"
+    PYINIT_ENTRY="-Wl,-ePyInit_${CHDB_PY_MOD}"
+    ICU="-DENABLE_ICU=1"
+    SED_INPLACE="sed -i"
+    # only x86_64, enable AVX, enable embedded compiler
+    if [ "$(uname -m)" == "x86_64" ]; then
+        CPU_FEATURES="-DENABLE_AVX=1 -DENABLE_AVX2=0"
+        LLVM="-DENABLE_EMBEDDED_COMPILER=1 -DENABLE_DWARF_PARSER=1"
+        RUST_FEATURES="-DENABLE_RUST=1 -DENABLE_DELTA_KERNEL_RS=1"
+        CORROSION_CMAKE_FILE="${PROJ_DIR}/contrib/corrosion-cmake/CMakeLists.txt"
+        if [ -f "${CORROSION_CMAKE_FILE}" ]; then
+            if ! grep -q 'OPENSSL_NO_DEPRECATED_3_0' "${CORROSION_CMAKE_FILE}"; then
+                echo "Modifying corrosion CMakeLists.txt for Linux x86_64..."
+                ${SED_INPLACE} 's/corrosion_set_env_vars(${target_name} "RUSTFLAGS=${RUSTFLAGS}")/corrosion_set_env_vars(${target_name} "RUSTFLAGS=${RUSTFLAGS} --cfg osslconf=\\\"OPENSSL_NO_DEPRECATED_3_0\\\"")/g' "${CORROSION_CMAKE_FILE}"
+            else
+                echo "corrosion CMakeLists.txt already modified, skipping..."
+            fi
+        else
+            echo "Warning: corrosion CMakeLists.txt not found at ${CORROSION_CMAKE_FILE}"
+        fi
+    else
+        CPU_FEATURES="-DENABLE_AVX=0 -DENABLE_AVX2=0 -DNO_ARMV81_OR_HIGHER=1"
+        LLVM="-DENABLE_EMBEDDED_COMPILER=0 -DENABLE_DWARF_PARSER=0"
+    fi
+else
+    echo "OS not supported"
+    exit 1
+fi
+
+if [ ! -d $BUILD_DIR ]; then
+    mkdir $BUILD_DIR
+fi
+
+cd ${BUILD_DIR}
+CMAKE_ARGS="-DCMAKE_BUILD_TYPE=${build_type} -DENABLE_THINLTO=0 -DENABLE_TESTS=0 -DENABLE_CLICKHOUSE_SERVER=0 -DENABLE_CLICKHOUSE_CLIENT=0 \
+    -DENABLE_CLICKHOUSE_KEEPER=0 -DENABLE_CLICKHOUSE_KEEPER_CONVERTER=0 -DENABLE_CLICKHOUSE_LOCAL=1 -DENABLE_CLICKHOUSE_SU=0 -DENABLE_CLICKHOUSE_BENCHMARK=0 \
+    -DENABLE_AZURE_BLOB_STORAGE=1 -DENABLE_CLICKHOUSE_COPIER=0 -DENABLE_CLICKHOUSE_DISKS=0 -DENABLE_CLICKHOUSE_FORMAT=0 -DENABLE_CLICKHOUSE_GIT_IMPORT=0 \
+    -DENABLE_AWS_S3=1 -DENABLE_HIVE=0 -DENABLE_AVRO=1 \
+    -DENABLE_CLICKHOUSE_OBFUSCATOR=0 -DENABLE_CLICKHOUSE_ODBC_BRIDGE=0 -DENABLE_CLICKHOUSE_STATIC_FILES_DISK_UPLOADER=0 \
+    -DENABLE_KAFKA=1 -DENABLE_LIBPQXX=1 -DENABLE_NATS=0 -DENABLE_AMQPCPP=0 -DENABLE_NURAFT=0 \
+    -DENABLE_CASSANDRA=0 -DENABLE_ODBC=0 -DENABLE_NLP=0 \
+    -DENABLE_LDAP=0 \
+    -DUSE_MUSL=1 \
+    -DRust_RUSTUP_INSTALL_MISSING_TARGET=ON \
+    ${MYSQL} \
+    ${HDFS} \
+    -DENABLE_LIBRARIES=0 ${RUST_FEATURES} \
+    ${GLIBC_COMPATIBILITY} \
+    -DENABLE_UTILS=0 ${LLVM} ${UNWIND} \
+    ${ICU} -DENABLE_UTF8PROC=1 ${JEMALLOC} \
+    -DENABLE_PARQUET=1 -DENABLE_ROCKSDB=1 -DENABLE_SQLITE=1 -DENABLE_VECTORSCAN=1 \
+    -DENABLE_PROTOBUF=1 -DENABLE_THRIFT=1 -DENABLE_MSGPACK=1 \
+    -DENABLE_BROTLI=1 -DENABLE_H3=1 -DENABLE_CURL=1 \
+    -DENABLE_CLICKHOUSE_ALL=0 -DUSE_STATIC_LIBRARIES=1 -DSPLIT_SHARED_LIBRARIES=0 \
+    -DENABLE_SIMDJSON=1 -DENABLE_RAPIDJSON=1 \
+    ${CPU_FEATURES} \
+    -DENABLE_AVX512=0 -DENABLE_AVX512_VBMI=0 \
+    -DENABLE_LIBFIU=1 \
+    ${COMPILER_CACHE} \
+    -DCHDB_VERSION=${CHDB_VERSION} \
+    "
+
+BINARY=${BUILD_DIR}/programs/clickhouse
+
+# build chdb python module
+py_version="3.8"
+current_py_version=$(python -c "import sys; print(f'{sys.version_info.major}.{sys.version_info.minor}')")
+if [ "$current_py_version" != "$py_version" ]; then
+    echo "Error: Current Python version is $current_py_version, but required version is $py_version"
+    echo "Please switch to Python $py_version using: pyenv shell $py_version"
+    exit 1
+fi
+echo "Using Python version: $current_py_version"
+cmake ${CMAKE_ARGS} -DENABLE_PYTHON=1 -DPYBIND11_NONLIMITEDAPI_PYTHON_HEADERS_VERSION=${py_version} ..
+ninja -d keeprsp || true
+
+# del the binary and run ninja -v again to capture the command, then modify it to generate CHDB_PY_MODULE
+/bin/rm -f ${BINARY}
+cd ${BUILD_DIR}
+ninja -d keeprsp -v > build.log || true
+
+USING_RESPONSE_FILE=$(grep -m 1 'clang++.*-o programs/clickhouse .*' build.log | grep '@CMakeFiles/clickhouse.rsp' || true)
+
+if [ ! "${USING_RESPONSE_FILE}" == "" ]; then
+    if [ -f CMakeFiles/clickhouse.rsp ]; then
+        cp -a CMakeFiles/clickhouse.rsp CMakeFiles/pychdb.rsp
+    else
+        echo "CMakeFiles/clickhouse.rsp not found"
+        exit 1
+    fi
+fi
+
+# extract the command to generate CHDB_PY_MODULE
+PYCHDB_CMD=$(grep -m 1 'clang++.*-o programs/clickhouse .*' build.log \
+    | sed "s/-o programs\/clickhouse/-fPIC -Wl,-undefined,dynamic_lookup -shared ${PYINIT_ENTRY} -o ${CHDB_PY_MODULE}/" \
+    | sed 's/^[^&]*&& //' | sed 's/&&.*//' \
+    | sed 's/ -Wl,-undefined,error/ -Wl,-undefined,dynamic_lookup/g' \
+    | sed 's/ -Xlinker --no-undefined//g' \
+    | sed 's/@CMakeFiles\/clickhouse.rsp/@CMakeFiles\/pychdb.rsp/g' \
+     )
+
+PYCHDB_CMD=$(echo ${PYCHDB_CMD} | sed 's/ src\/CMakeFiles\/clickhouse_malloc.dir\/Common\/stubFree.c.o//g')
+if [ ! "${USING_RESPONSE_FILE}" == "" ]; then
+    ${SED_INPLACE} 's/ src\/CMakeFiles\/clickhouse_malloc.dir\/Common\/stubFree.c.o//g' CMakeFiles/pychdb.rsp
+fi
+
+PYCHDB_CMD=$(echo ${PYCHDB_CMD} | sed 's|-Wl,-rpath,/[^[:space:]]*/pybind11-cmake|-Wl,-rpath,\$ORIGIN|g')
+
+echo ${PYCHDB_CMD} > pychdb_cmd.sh
+
+${PYCHDB_CMD}
+
+ls -lh ${CHDB_PY_MODULE}
+
+PYCHDB=${BUILD_DIR}/${CHDB_PY_MODULE}
+
+if [ ${build_type} == "Debug" ]; then
+    echo -e "\nDebug build, skip strip"
+else
+    echo -e "\nStrip the binary:"
+    ${STRIP} --remove-section=.comment --remove-section=.note ${PYCHDB}
+fi
+echo -e "\nStripe the binary:"
+
+echo -e "\nPYCHDB: ${PYCHDB}"
+ls -lh ${PYCHDB}
+echo -e "\nldd ${PYCHDB}"
+${LDD} ${PYCHDB} || echo "Binary is statically linked (not a dynamic executable)"
+echo -e "\nfile info of ${PYCHDB}"
+file ${PYCHDB}
+
+rm -f ${CHDB_DIR}/*.so
+cp -a ${PYCHDB} ${CHDB_DIR}/${CHDB_PY_MODULE}
+
+echo -e "\nSymbols:"
+echo -e "\nPyInit in PYCHDB: ${PYCHDB}"
+${NM} ${PYCHDB} | grep PyInit || true
+echo -e "\nquery_stable in PYCHDB: ${PYCHDB}"
+${NM} ${PYCHDB} | grep query_stable || true
+
+echo -e "\nAfter copy:"
+cd ${PROJ_DIR} && pwd
+
+ccache -s || true
+
+CMAKE_ARGS="${CMAKE_ARGS}" bash ${DIR}/build_pybind11.sh --all