chdb 3.7.1__cp38-abi3-musllinux_1_2_aarch64.whl

chdb/udf/__init__.py

@@ -0,0 +1,10 @@
+"""User-defined functions module for chDB.
+
+This module provides functionality for creating and managing user-defined functions (UDFs)
+in chDB. It allows you to extend chDB's capabilities by writing custom Python functions
+that can be called from SQL queries.
+"""
+
+from .udf import chdb_udf, generate_udf
+
+__all__ = ["chdb_udf", "generate_udf"]

chdb/udf/udf.py

@@ -0,0 +1,122 @@
+import functools
+import inspect
+import os
+import sys
+import tempfile
+import atexit
+import shutil
+import textwrap
+from xml.etree import ElementTree as ET
+import chdb
+
+
+def generate_udf(func_name, args, return_type, udf_body):
+    """Generate UDF configuration and executable script files.
+
+    This function creates the necessary files for a User Defined Function (UDF) in chDB:
+    1. A Python executable script that processes input data
+    2. An XML configuration file that registers the UDF with ClickHouse
+
+    Args:
+        func_name (str): Name of the UDF function
+        args (list): List of argument names for the function
+        return_type (str): ClickHouse return type for the function
+        udf_body (str): Python source code body of the UDF function
+
+    Note:
+        This function is typically called by the @chdb_udf decorator and should not
+        be called directly by users.
+    """
+    # generate python script
+    with open(f"{chdb.g_udf_path}/{func_name}.py", "w") as f:
+        f.write(f"#!{sys.executable}\n")
+        f.write("import sys\n")
+        f.write("\n")
+        for line in udf_body.split("\n"):
+            f.write(f"{line}\n")
+        f.write("\n")
+        f.write("if __name__ == '__main__':\n")
+        f.write("    for line in sys.stdin:\n")
+        f.write("        args = line.strip().split('\t')\n")
+        for i, arg in enumerate(args):
+            f.write(f"        {arg} = args[{i}]\n")
+        f.write(f"        print({func_name}({', '.join(args)}))\n")
+        f.write("        sys.stdout.flush()\n")
+    os.chmod(f"{chdb.g_udf_path}/{func_name}.py", 0o755)
+    # generate xml file
+    xml_file = f"{chdb.g_udf_path}/udf_config.xml"
+    root = ET.Element("functions")
+    if os.path.exists(xml_file):
+        tree = ET.parse(xml_file)
+        root = tree.getroot()
+    function = ET.SubElement(root, "function")
+    ET.SubElement(function, "type").text = "executable"
+    ET.SubElement(function, "name").text = func_name
+    ET.SubElement(function, "return_type").text = return_type
+    ET.SubElement(function, "format").text = "TabSeparated"
+    ET.SubElement(function, "command").text = f"{func_name}.py"
+    for arg in args:
+        argument = ET.SubElement(function, "argument")
+        # We use TabSeparated format, so assume all arguments are strings
+        ET.SubElement(argument, "type").text = "String"
+        ET.SubElement(argument, "name").text = arg
+    tree = ET.ElementTree(root)
+    tree.write(xml_file)
+
+
+def chdb_udf(return_type="String"):
+    """Decorator for chDB Python UDF(User Defined Function).
+
+    Args:
+        return_type (str): Return type of the function. Default is "String".
+            Should be one of the ClickHouse data types.
+
+    Notes:
+        1. The function should be stateless. Only UDFs are supported, not UDAFs.
+        2. Default return type is String. The return type should be one of the ClickHouse data types.
+        3. The function should take in arguments of type String. All arguments are strings.
+        4. The function will be called for each line of input.
+        5. The function should be pure python function. Import all modules used IN THE FUNCTION.
+        6. Python interpreter used is the same as the one used to run the script.
+
+    Example:
+        .. code-block:: python
+
+            @chdb_udf()
+            def sum_udf(lhs, rhs):
+                return int(lhs) + int(rhs)
+
+            @chdb_udf()
+            def func_use_json(arg):
+                import json
+                # ... use json module
+    """
+
+    def decorator(func):
+        func_name = func.__name__
+        sig = inspect.signature(func)
+        args = list(sig.parameters.keys())
+        src = inspect.getsource(func)
+        src = textwrap.dedent(src)
+        udf_body = src.split("\n", 1)[1]  # remove the first line "@chdb_udf()"
+        # create tmp dir and make sure the dir is deleted when the process exits
+        if chdb.g_udf_path == "":
+            chdb.g_udf_path = tempfile.mkdtemp()
+
+        # clean up the tmp dir on exit
+        @atexit.register
+        def _cleanup():
+            try:
+                shutil.rmtree(chdb.g_udf_path)
+            except:  # noqa
+                pass
+
+        generate_udf(func_name, args, return_type, udf_body)
+
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs):
+            return func(*args, **kwargs)
+
+        return wrapper
+
+    return decorator

chdb/utils/__init__.py

@@ -0,0 +1,15 @@
+"""Utility functions and helpers for chDB.
+
+This module contains various utility functions for working with chDB, including
+data type inference, data conversion helpers, and debugging utilities.
+"""
+
+from .types import *  # noqa: F403
+
+__all__ = [  # noqa: F405
+    "flatten_dict",
+    "convert_to_columnar",
+    "infer_data_type",
+    "infer_data_types",
+    "trace",
+]

chdb/utils/trace.py

@@ -0,0 +1,105 @@
+import functools
+import inspect
+import sys
+import linecache
+from datetime import datetime
+
+enable_print = False
+
+
+def trace(func):
+    """Trace function execution with line-by-line debugging information.
+
+    This decorator enables line-by-line tracing of function execution, printing
+    each executed line along with local variable values. Tracing is only enabled
+    when the global `enable_print` variable is True.
+
+    Args:
+        func: The function to be traced
+
+    Returns:
+        The wrapped function with tracing capabilities
+
+    Note:
+        Set `enable_print = True` to enable tracing output. When disabled,
+        the function executes normally without any performance overhead.
+
+    Example:
+        .. code-block:: python
+
+            import chdb.utils.trace
+            chdb.utils.trace.enable_print = True
+
+            @chdb.utils.trace.trace
+            def my_function(x, y):
+                z = x + y
+                return z * 2
+    """
+    return print_lines(func)
+
+
+def print_lines(func):
+    if not enable_print:
+        return func
+
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+        # Get function name and determine if it's a method
+        is_method = inspect.ismethod(func) or (
+            len(args) > 0 and hasattr(args[0].__class__, func.__name__)
+        )
+        class_name = args[0].__class__.__name__ if is_method else None  # type: ignore
+
+        # Get the source code of the function
+        try:
+            source_lines, start_line = inspect.getsourcelines(func)
+        except OSError:
+            # Handle cases where source might not be available
+            print(f"Warning: Could not get source for {func.__name__}")
+            return func(*args, **kwargs)
+
+        def trace(frame, event, arg):
+            if event == "line":
+                # Get the current line number and code
+                line_no = frame.f_lineno
+                line = linecache.getline(frame.f_code.co_filename, line_no).strip()
+
+                # Don't print decorator lines or empty lines
+                if line and not line.startswith("@"):
+                    # Get local variables
+                    local_vars = frame.f_locals.copy()
+                    if is_method:
+                        # Remove 'self' from local variables for clarity
+                        local_vars.pop("self", None)
+
+                    # Format timestamp
+                    timestamp = datetime.now().strftime("%H:%M:%S.%f")[:-3]
+
+                    # Create context string (class.method or function)
+                    context = (
+                        f"{class_name}.{func.__name__}" if class_name else func.__name__
+                    )
+
+                    # Print execution information
+                    print(f"[{timestamp}] {context} line {line_no}: {line}")
+
+                    # Print local variables if they exist and have changed
+                    if local_vars:
+                        vars_str = ", ".join(
+                            f"{k}={repr(v)}" for k, v in local_vars.items()
+                        )
+                        print(f"    Variables: {vars_str}")
+            return trace
+
+        # Set the trace function
+        sys.settrace(trace)
+
+        # Call the original function
+        result = func(*args, **kwargs)
+
+        # Disable tracing
+        sys.settrace(None)
+
+        return result
+
+    return wrapper

chdb/utils/types.py

@@ -0,0 +1,232 @@
+from collections import defaultdict
+from typing import List, Dict, Any
+import json
+import decimal
+
+
+def convert_to_columnar(items: List[Dict[str, Any]]) -> Dict[str, List[Any]]:
+    """Converts a list of dictionaries into a columnar format.
+
+    This function takes a list of dictionaries and converts it into a dictionary
+    where each key corresponds to a column and each value is a list of column values.
+    Missing values in the dictionaries are represented as None.
+
+    Args:
+        items (List[Dict[str, Any]]): A list of dictionaries to convert.
+
+    Returns:
+        Dict[str, List[Any]]: A dictionary with keys as column names and values as lists
+            of column values.
+
+    Example:
+        >>> items = [
+        ...     {"name": "Alice", "age": 30, "city": "New York"},
+        ...     {"name": "Bob", "age": 25},
+        ...     {"name": "Charlie", "city": "San Francisco"}
+        ... ]
+        >>> convert_to_columnar(items)
+        {
+            'name': ['Alice', 'Bob', 'Charlie'],
+            'age': [30, 25, None],
+            'city': ['New York', None, 'San Francisco']
+        }
+    """
+    if not items:
+        return {}
+
+    flattened_items = [flatten_dict(item) for item in items]
+    columns = defaultdict(list)
+    keys = set()
+
+    # Collect all possible keys
+    for flattened_item in flattened_items:
+        keys.update(flattened_item.keys())
+
+    # Fill the column lists
+    for flattened_item in flattened_items:
+        for key in keys:
+            columns[key].append(flattened_item.get(key, None))
+
+    return dict(columns)
+
+
+def flatten_dict(
+    d: Dict[str, Any], parent_key: str = "", sep: str = "_"
+) -> Dict[str, Any]:
+    """Flattens a nested dictionary.
+
+    This function takes a nested dictionary and flattens it, concatenating nested keys
+    with a separator. Lists of dictionaries are serialized to JSON strings.
+
+    Args:
+        d (Dict[str, Any]): The dictionary to flatten.
+        parent_key (str, optional): The base key to prepend to each key. Defaults to "".
+        sep (str, optional): The separator to use between concatenated keys. Defaults to "_".
+
+    Returns:
+        Dict[str, Any]: A flattened dictionary.
+
+    Example:
+        >>> nested_dict = {
+        ...     "a": 1,
+        ...     "b": {
+        ...         "c": 2,
+        ...         "d": {
+        ...             "e": 3
+        ...         }
+        ...     },
+        ...     "f": [4, 5, {"g": 6}],
+        ...     "h": [{"i": 7}, {"j": 8}]
+        ... }
+        >>> flatten_dict(nested_dict)
+        {
+            'a': 1,
+            'b_c': 2,
+            'b_d_e': 3,
+            'f_0': 4,
+            'f_1': 5,
+            'f_2_g': 6,
+            'h': '[{"i": 7}, {"j": 8}]'
+        }
+    """
+    items = []
+    for k, v in d.items():
+        new_key = f"{parent_key}{sep}{k}" if parent_key else k
+        if isinstance(v, dict):
+            items.extend(flatten_dict(v, new_key, sep=sep).items())
+        elif isinstance(v, list):
+            if all(isinstance(i, dict) for i in v):
+                items.append((new_key, json.dumps(v)))
+            else:
+                for i, item in enumerate(v):
+                    if isinstance(item, dict):
+                        items.extend(
+                            flatten_dict(item, f"{new_key}{sep}{i}", sep=sep).items()
+                        )
+                    else:
+                        items.append((f"{new_key}{sep}{i}", item))
+        else:
+            items.append((new_key, v))
+    return dict(items)
+
+
+def infer_data_types(
+    column_data: Dict[str, List[Any]], n_rows: int = 10000
+) -> List[tuple]:
+    """Infers data types for each column in a columnar data structure.
+
+    This function analyzes the values in each column and infers the most suitable
+    data type for each column, based on a sample of the data.
+
+    Args:
+        column_data (Dict[str, List[Any]]): A dictionary where keys are column names
+            and values are lists of column values.
+        n_rows (int, optional): The number of rows to sample for type inference.
+            Defaults to 10000.
+
+    Returns:
+        List[tuple]: A list of tuples, each containing a column name and its
+            inferred data type.
+    """
+    data_types = []
+    for column, values in column_data.items():
+        sampled_values = values[:n_rows]
+        inferred_type = infer_data_type(sampled_values)
+        data_types.append((column, inferred_type))
+    return data_types
+
+
+def infer_data_type(values: List[Any]) -> str:
+    """Infers the most suitable data type for a list of values.
+
+    This function examines a list of values and determines the most appropriate
+    data type that can represent all the values in the list. It considers integer,
+    unsigned integer, decimal, and float types, and defaults to "string" if the
+    values cannot be represented by any numeric type or if all values are None.
+
+    Args:
+        values (List[Any]): A list of values to analyze. The values can be of any type.
+
+    Returns:
+        str: A string representing the inferred data type. Possible return values are:
+            "int8", "int16", "int32", "int64", "int128", "int256", "uint8", "uint16",
+            "uint32", "uint64", "uint128", "uint256", "decimal128", "decimal256",
+            "float32", "float64", or "string".
+
+    Notes:
+        - If all values in the list are None, the function returns "string".
+        - If any value in the list is a string, the function immediately returns "string".
+        - The function assumes that numeric values can be represented as integers,
+          decimals, or floats based on their range and precision.
+    """
+
+    int_range = {
+        "int8": (-(2**7), 2**7 - 1),
+        "int16": (-(2**15), 2**15 - 1),
+        "int32": (-(2**31), 2**31 - 1),
+        "int64": (-(2**63), 2**63 - 1),
+        "int128": (-(2**127), 2**127 - 1),
+        "int256": (-(2**255), 2**255 - 1),
+    }
+    uint_range = {
+        "uint8": (0, 2**8 - 1),
+        "uint16": (0, 2**16 - 1),
+        "uint32": (0, 2**32 - 1),
+        "uint64": (0, 2**64 - 1),
+        "uint128": (0, 2**128 - 1),
+        "uint256": (0, 2**256 - 1),
+    }
+
+    max_val = float("-inf")
+    min_val = float("inf")
+    is_int = True
+    is_decimal = True
+    is_float = True
+
+    all_none = True
+
+    for val in values:
+        if val is None:
+            continue
+        all_none = False
+        if isinstance(val, str):
+            return "string"
+
+        try:
+            num = int(val)
+            max_val = max(max_val, num)
+            min_val = min(min_val, num)
+        except (ValueError, TypeError):
+            is_int = False
+            try:
+                num = decimal.Decimal(val)
+                max_val = max(max_val, float(num))
+                min_val = min(min_val, float(num))
+            except (decimal.InvalidOperation, TypeError):
+                is_decimal = False
+                try:
+                    num = float(val)
+                    max_val = max(max_val, num)
+                    min_val = min(min_val, num)
+                except (ValueError, TypeError):
+                    is_float = False
+                    return "string"
+
+    if all_none:
+        return "string"
+
+    if is_int:
+        for dtype, (min_val_dtype, max_val_dtype) in int_range.items():
+            if min_val_dtype <= min_val and max_val <= max_val_dtype:
+                return dtype
+        for dtype, (_, max_val_dtype) in uint_range.items():
+            if max_val <= max_val_dtype:
+                return dtype
+
+    if is_decimal:
+        return "decimal128" if abs(max_val) < 10**38 else "decimal256"
+
+    if is_float:
+        return "float32" if abs(max_val) < 3.4e38 else "float64"
+
+    return "string"

chdb-3.7.1.dist-info/LICENSE.txt

@@ -0,0 +1,203 @@
+Copyright 2023 chDB, Inc.
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2023 chDB, Inc.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.