singlestoredb 1.0.4__cp38-abi3-macosx_10_9_universal2.whl → 1.2.0__cp38-abi3-macosx_10_9_universal2.whl

singlestoredb/__init__.py

@@ -13,7 +13,7 @@ Examples
 
 """
 
-__version__ = '1.0.4'
+__version__ = '1.2.0'
 
 from typing import Any
 

singlestoredb/config.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 """SingleStoreDB package options."""
 import functools
+import os
 
 from . import auth
 from .utils.config import check_bool  # noqa: F401
@@ -214,6 +215,12 @@ register_option(
     environ='SINGLESTOREDB_TRACK_ENV',
 )
 
+register_option(
+    'enable_extended_data_types', 'bool', check_bool, True,
+    'Should extended data types (BSON, vector) be enabled?',
+    environ='SINGLESTOREDB_ENABLE_EXTENDED_DATA_TYPES',
+)
+
 register_option(
     'fusion.enabled', 'bool', check_bool, False,
     'Should Fusion SQL queries be enabled?',
@@ -230,6 +237,7 @@ register_option(
         valid_values=[
             'tuple', 'tuples', 'namedtuple', 'namedtuples',
             'dict', 'dicts', 'structsequence', 'structsequences',
+            'numpy', 'pandas', 'polars', 'arrow', 'pyarrow',
         ],
     ),
     'tuples',
@@ -266,6 +274,129 @@ register_option(
 )
 
 
+#
+# External function options
+#
+register_option(
+    'external_function.url', 'string', check_str, 'http://localhost:8000/invoke',
+    'Specifies the URL of the external function application.',
+    environ=['SINGLESTOREDB_EXT_FUNC_URL'],
+)
+
+register_option(
+    'external_function.app_mode', 'string',
+    functools.partial(
+        check_str,
+        valid_values=['remote', 'collocated'],
+    ),
+    'remote',
+    'Specifies the mode of operation of the external function application.',
+    environ=['SINGLESTOREDB_EXT_FUNC_APP_MODE'],
+)
+
+register_option(
+    'external_function.data_format', 'string',
+    functools.partial(
+        check_str,
+        valid_values=['rowdat_1', 'json'],
+    ),
+    'rowdat_1',
+    'Specifies the format for the data rows.',
+    environ=['SINGLESTOREDB_EXT_FUNC_DATA_FORMAT'],
+)
+
+register_option(
+    'external_function.data_version', 'string', check_str, '1.0',
+    'Specifies the version of the data format.',
+    environ=['SINGLESTOREDB_EXT_FUNC_DATA_VERSION'],
+)
+
+register_option(
+    'external_function.link_name', 'string', check_str, None,
+    'Specifies the link name to use for remote external functions.',
+    environ=['SINGLESTOREDB_EXT_FUNC_LINK_NAME'],
+)
+
+register_option(
+    'external_function.link_config', 'string', check_str, None,
+    'Specifies the link config in JSON format.',
+    environ=['SINGLESTOREDB_EXT_FUNC_LINK_CONFIG'],
+)
+
+register_option(
+    'external_function.link_credentials', 'string', check_str, None,
+    'Specifies the link credentials in JSON format.',
+    environ=['SINGLESTOREDB_EXT_FUNC_LINK_CREDENTIALS'],
+)
+
+register_option(
+    'external_function.replace_existing', 'bool', check_bool, False,
+    'Should existing functions be replaced when registering external functions?',
+    environ=['SINGLESTOREDB_EXT_FUNC_REPLACE_EXISTING'],
+)
+
+register_option(
+    'external_function.socket_path', 'string', check_str, None,
+    'Specifies the socket path for collocated external functions.',
+    environ=['SINGLESTOREDB_EXT_FUNC_SOCKET_PATH'],
+)
+
+register_option(
+    'external_function.max_connections', 'int', check_int, 32,
+    'Specifies the maximum connections in a collocated external function ' +
+    'before reusing them.',
+    environ=['SINGLESTOREDB_EXT_FUNC_MAX_CONNECTIONS'],
+)
+
+register_option(
+    'external_function.process_mode', 'string',
+    functools.partial(
+        check_str,
+        valid_values=['thread', 'subprocess'],
+    ),
+    'subprocess',
+    'Specifies the method to use for concurrent handlers in ' +
+    'collocated external functions',
+    environ=['SINGLESTOREDB_EXT_FUNC_PROCESS_MODE'],
+)
+
+register_option(
+    'external_function.single_thread', 'bool', check_bool, False,
+    'Should the collocated server run in single-thread mode?',
+    environ=['SINGLESTOREDB_EXT_FUNC_SINGLE_THREAD'],
+)
+
+register_option(
+    'external_function.log_level', 'string',
+    functools.partial(
+        check_str,
+        valid_values=['info', 'debug', 'warning', 'error'],
+    ),
+    'info',
+    'Logging level of external function server.',
+    environ=['SINGLESTOREDB_EXT_FUNC_LOG_LEVEL'],
+)
+
+register_option(
+    'external_function.connection', 'string', check_str,
+    os.environ.get('SINGLESTOREDB_URL') or None,
+    'Specifies the connection string for the database to register functions with.',
+    environ=['SINGLESTOREDB_EXT_FUNC_CONNECTION'],
+)
+
+register_option(
+    'external_function.host', 'string', check_str, '127.0.0.1',
+    'Specifies the host to bind the server to.',
+    environ=['SINGLESTOREDB_EXT_FUNC_HOST'],
+)
+
+register_option(
+    'external_function.port', 'int', check_int, 8000,
+    'Specifies the port to bind the server to.',
+    environ=['SINGLESTOREDB_EXT_FUNC_PORT'],
+)
+
+
 #
 # Debugging options
 #

singlestoredb/connection.py

@@ -1285,6 +1285,7 @@ def connect(
     inf_as_null: Optional[bool] = None,
     encoding_errors: Optional[str] = None,
     track_env: Optional[bool] = None,
+    enable_extended_data_types: Optional[bool] = None,
 ) -> Connection:
     """
     Return a SingleStoreDB connection.
@@ -1361,6 +1362,8 @@ def connect(
         The error handler name for value decoding errors
     track_env : bool, optional
         Should the connection track the SINGLESTOREDB_URL environment variable?
+    enable_extended_data_types : bool, optional
+        Should extended data types (BSON, vector) be enabled?
 
     Examples
     --------

singlestoredb/converters.py

@@ -2,6 +2,7 @@
 """Data value conversion utilities."""
 import datetime
 import re
+import struct
 from base64 import b64decode
 from decimal import Decimal
 from json import loads as json_loads
@@ -25,7 +26,20 @@ try:
 except ImportError:
     has_pygeos = False
 
+try:
+    import numpy
+    has_numpy = True
+except ImportError:
+    has_numpy = False
+
+try:
+    import bson
+    has_bson = True
+except ImportError:
+    has_bson = False
+
 
+# Cache fromisoformat methods if they exist
 # Cache fromisoformat methods if they exist
 _dt_datetime_fromisoformat = None
 if hasattr(datetime.datetime, 'fromisoformat'):
@@ -527,6 +541,369 @@ def geometry_or_none(x: Optional[str]) -> Optional[Any]:
     return x
 
 
+def float32_vector_json_or_none(x: Optional[str]) -> Optional[Any]:
+    """
+    Covert value to float32 array.
+
+    Parameters
+    ----------
+    x : str or None
+        JSON array
+
+    Returns
+    -------
+    float32 numpy array
+        If input value is not None and numpy is installed
+    float Python list
+        If input value is not None and numpy is not installed
+    None
+        If input value is None
+
+    """
+    if x is None:
+        return None
+
+    if has_numpy:
+        return numpy.array(json_loads(x), dtype=numpy.float32)
+
+    return map(float, json_loads(x))
+
+
+def float32_vector_or_none(x: Optional[bytes]) -> Optional[Any]:
+    """
+    Covert value to float32 array.
+
+    Parameters
+    ----------
+    x : bytes or None
+        Little-endian block of bytes.
+
+    Returns
+    -------
+    float32 numpy array
+        If input value is not None and numpy is installed
+    float Python list
+        If input value is not None and numpy is not installed
+    None
+        If input value is None
+
+    """
+    if x is None:
+        return None
+
+    if has_numpy:
+        return numpy.frombuffer(x, dtype=numpy.float32)
+
+    return struct.unpack(f'<{len(x)/4}f', x)
+
+
+def float64_vector_json_or_none(x: Optional[str]) -> Optional[Any]:
+    """
+    Covert value to float64 array.
+
+    Parameters
+    ----------
+    x : str or None
+        JSON array
+
+    Returns
+    -------
+    float64 numpy array
+        If input value is not None and numpy is installed
+    float Python list
+        If input value is not None and numpy is not installed
+    None
+        If input value is None
+
+    """
+    if x is None:
+        return None
+
+    if has_numpy:
+        return numpy.array(json_loads(x), dtype=numpy.float64)
+
+    return map(float, json_loads(x))
+
+
+def float64_vector_or_none(x: Optional[bytes]) -> Optional[Any]:
+    """
+    Covert value to float64 array.
+
+    Parameters
+    ----------
+    x : bytes or None
+        JSON array
+
+    Returns
+    -------
+    float64 numpy array
+        If input value is not None and numpy is installed
+    float Python list
+        If input value is not None and numpy is not installed
+    None
+        If input value is None
+
+    """
+    if x is None:
+        return None
+
+    if has_numpy:
+        return numpy.frombuffer(x, dtype=numpy.float64)
+
+    return struct.unpack(f'<{len(x)/8}d', x)
+
+
+def int8_vector_json_or_none(x: Optional[str]) -> Optional[Any]:
+    """
+    Covert value to int8 array.
+
+    Parameters
+    ----------
+    x : str or None
+        JSON array
+
+    Returns
+    -------
+    int8 numpy array
+        If input value is not None and numpy is installed
+    int Python list
+        If input value is not None and numpy is not installed
+    None
+        If input value is None
+
+    """
+    if x is None:
+        return None
+
+    if has_numpy:
+        return numpy.array(json_loads(x), dtype=numpy.int8)
+
+    return map(int, json_loads(x))
+
+
+def int8_vector_or_none(x: Optional[bytes]) -> Optional[Any]:
+    """
+    Covert value to int8 array.
+
+    Parameters
+    ----------
+    x : bytes or None
+        Little-endian block of bytes.
+
+    Returns
+    -------
+    int8 numpy array
+        If input value is not None and numpy is installed
+    int Python list
+        If input value is not None and numpy is not installed
+    None
+        If input value is None
+
+    """
+    if x is None:
+        return None
+
+    if has_numpy:
+        return numpy.frombuffer(x, dtype=numpy.int8)
+
+    return struct.unpack(f'<{len(x)}b', x)
+
+
+def int16_vector_json_or_none(x: Optional[str]) -> Optional[Any]:
+    """
+    Covert value to int16 array.
+
+    Parameters
+    ----------
+    x : str or None
+        JSON array
+
+    Returns
+    -------
+    int16 numpy array
+        If input value is not None and numpy is installed
+    int Python list
+        If input value is not None and numpy is not installed
+    None
+        If input value is None
+
+    """
+    if x is None:
+        return None
+
+    if has_numpy:
+        return numpy.array(json_loads(x), dtype=numpy.int16)
+
+    return map(int, json_loads(x))
+
+
+def int16_vector_or_none(x: Optional[bytes]) -> Optional[Any]:
+    """
+    Covert value to int16 array.
+
+    Parameters
+    ----------
+    x : bytes or None
+        Little-endian block of bytes.
+
+    Returns
+    -------
+    int16 numpy array
+        If input value is not None and numpy is installed
+    int Python list
+        If input value is not None and numpy is not installed
+    None
+        If input value is None
+
+    """
+    if x is None:
+        return None
+
+    if has_numpy:
+        return numpy.frombuffer(x, dtype=numpy.int16)
+
+    return struct.unpack(f'<{len(x)/2}h', x)
+
+
+def int32_vector_json_or_none(x: Optional[str]) -> Optional[Any]:
+    """
+    Covert value to int32 array.
+
+    Parameters
+    ----------
+    x : str or None
+        JSON array
+
+    Returns
+    -------
+    int32 numpy array
+        If input value is not None and numpy is installed
+    int Python list
+        If input value is not None and numpy is not installed
+    None
+        If input value is None
+
+    """
+    if x is None:
+        return None
+
+    if has_numpy:
+        return numpy.array(json_loads(x), dtype=numpy.int32)
+
+    return map(int, json_loads(x))
+
+
+def int32_vector_or_none(x: Optional[bytes]) -> Optional[Any]:
+    """
+    Covert value to int32 array.
+
+    Parameters
+    ----------
+    x : bytes or None
+        Little-endian block of bytes.
+
+    Returns
+    -------
+    int32 numpy array
+        If input value is not None and numpy is installed
+    int Python list
+        If input value is not None and numpy is not installed
+    None
+        If input value is None
+
+    """
+    if x is None:
+        return None
+
+    if has_numpy:
+        return numpy.frombuffer(x, dtype=numpy.int32)
+
+    return struct.unpack(f'<{len(x)/4}l', x)
+
+
+def int64_vector_json_or_none(x: Optional[str]) -> Optional[Any]:
+    """
+    Covert value to int64 array.
+
+    Parameters
+    ----------
+    x : str or None
+        JSON array
+
+    Returns
+    -------
+    int64 numpy array
+        If input value is not None and numpy is installed
+    int Python list
+        If input value is not None and numpy is not installed
+    None
+        If input value is None
+
+    """
+    if x is None:
+        return None
+
+    if has_numpy:
+        return numpy.array(json_loads(x), dtype=numpy.int64)
+
+    return map(int, json_loads(x))
+
+
+def int64_vector_or_none(x: Optional[bytes]) -> Optional[Any]:
+    """
+    Covert value to int64 array.
+
+    Parameters
+    ----------
+    x : bytes or None
+        Little-endian block of bytes.
+
+    Returns
+    -------
+    int64 numpy array
+        If input value is not None and numpy is installed
+    int Python list
+        If input value is not None and numpy is not installed
+    None
+        If input value is None
+
+    """
+    if x is None:
+        return None
+
+    # Bytes
+    if has_numpy:
+        return numpy.frombuffer(x, dtype=numpy.int64)
+
+    return struct.unpack(f'<{len(x)/8}l', x)
+
+
+def bson_or_none(x: Optional[bytes]) -> Optional[Any]:
+    """
+    Convert a BSON value to a dictionary.
+
+    Parameters
+    ----------
+    x : bytes or None
+        BSON formatted bytes
+
+    Returns
+    -------
+    dict
+        If input value is not None and bson package is installed
+    bytes
+        If input value is not None and bson package is not installed
+    None
+        If input value is None
+
+    """
+    if x is None:
+        return None
+    if has_bson:
+        return bson.decode(x)
+    return x
+
+
 # Map of database types and conversion functions
 converters: Dict[int, Callable[..., Any]] = {
     0: decimal_or_none,
@@ -557,4 +934,17 @@ converters: Dict[int, Callable[..., Any]] = {
     #   253: identity,
     #   254: identity,
     255: geometry_or_none,
+    1001: bson_or_none,
+    2001: float32_vector_json_or_none,
+    2002: float64_vector_json_or_none,
+    2003: int8_vector_json_or_none,
+    2004: int16_vector_json_or_none,
+    2005: int32_vector_json_or_none,
+    2006: int64_vector_json_or_none,
+    3001: float32_vector_or_none,
+    3002: float64_vector_or_none,
+    3003: int8_vector_or_none,
+    3004: int16_vector_or_none,
+    3005: int32_vector_or_none,
+    3006: int64_vector_or_none,
 }