singlestoredb 1.1.0__py3-none-any.whl → 1.3.0__py3-none-any.whl

singlestoredb/__init__.py

@@ -13,7 +13,7 @@ Examples
 
 """
 
-__version__ = '1.1.0'
+__version__ = '1.3.0'
 
 from typing import Any
 

singlestoredb/config.py

@@ -215,6 +215,12 @@ register_option(
     environ='SINGLESTOREDB_TRACK_ENV',
 )
 
+register_option(
+    'enable_extended_data_types', 'bool', check_bool, True,
+    'Should extended data types (BSON, vector) be enabled?',
+    environ='SINGLESTOREDB_ENABLE_EXTENDED_DATA_TYPES',
+)
+
 register_option(
     'fusion.enabled', 'bool', check_bool, False,
     'Should Fusion SQL queries be enabled?',

singlestoredb/connection.py

@@ -2,7 +2,10 @@
 """SingleStoreDB connections and cursors."""
 import abc
 import inspect
+import io
+import queue
 import re
+import sys
 import warnings
 import weakref
 from collections.abc import Mapping
@@ -33,6 +36,11 @@ from .config import get_option
 from .utils.results import Description
 from .utils.results import Result
 
+if sys.version_info < (3, 10):
+    InfileQueue = queue.Queue
+else:
+    InfileQueue = queue.Queue[Union[bytes, str]]
+
 
 # DB-API settings
 apilevel = '2.0'
@@ -496,6 +504,14 @@ class Cursor(metaclass=abc.ABCMeta):
     def execute(
         self, query: str,
         args: Optional[Union[Sequence[Any], Dict[str, Any], Any]] = None,
+        infile_stream: Optional[  # type: ignore
+            Union[
+                io.RawIOBase,
+                io.TextIOBase,
+                Iterator[Union[bytes, str]],
+                InfileQueue,
+            ]
+        ] = None,
     ) -> int:
         """
         Execute a SQL statement.
@@ -510,6 +526,8 @@ class Cursor(metaclass=abc.ABCMeta):
             The SQL statement to execute
         args : Sequence or dict, optional
             Parameters to substitute into the SQL code
+        infile_stream : io.RawIOBase or io.TextIOBase or Iterator[bytes|str], optional
+            Data stream for ``LOCAL INFILE`` statement
 
         Examples
         --------
@@ -1285,6 +1303,7 @@ def connect(
     inf_as_null: Optional[bool] = None,
     encoding_errors: Optional[str] = None,
     track_env: Optional[bool] = None,
+    enable_extended_data_types: Optional[bool] = None,
 ) -> Connection:
     """
     Return a SingleStoreDB connection.
@@ -1338,7 +1357,8 @@ def connect(
     autocommit : bool, optional
         Enable autocommits
     results_type : str, optional
-        The form of the query results: tuples, namedtuples, dicts
+        The form of the query results: tuples, namedtuples, dicts,
+        numpy, polars, pandas, arrow
     results_format : str, optional
         Deprecated. This option has been renamed to results_type.
     program_name : str, optional
@@ -1361,6 +1381,8 @@ def connect(
         The error handler name for value decoding errors
     track_env : bool, optional
         Should the connection track the SINGLESTOREDB_URL environment variable?
+    enable_extended_data_types : bool, optional
+        Should extended data types (BSON, vector) be enabled?
 
     Examples
     --------

singlestoredb/converters.py

@@ -2,6 +2,7 @@
 """Data value conversion utilities."""
 import datetime
 import re
+import struct
 from base64 import b64decode
 from decimal import Decimal
 from json import loads as json_loads
@@ -25,7 +26,20 @@ try:
 except ImportError:
     has_pygeos = False
 
+try:
+    import numpy
+    has_numpy = True
+except ImportError:
+    has_numpy = False
+
+try:
+    import bson
+    has_bson = True
+except ImportError:
+    has_bson = False
+
 
+# Cache fromisoformat methods if they exist
 # Cache fromisoformat methods if they exist
 _dt_datetime_fromisoformat = None
 if hasattr(datetime.datetime, 'fromisoformat'):
@@ -527,6 +541,369 @@ def geometry_or_none(x: Optional[str]) -> Optional[Any]:
     return x
 
 
+def float32_vector_json_or_none(x: Optional[str]) -> Optional[Any]:
+    """
+    Covert value to float32 array.
+
+    Parameters
+    ----------
+    x : str or None
+        JSON array
+
+    Returns
+    -------
+    float32 numpy array
+        If input value is not None and numpy is installed
+    float Python list
+        If input value is not None and numpy is not installed
+    None
+        If input value is None
+
+    """
+    if x is None:
+        return None
+
+    if has_numpy:
+        return numpy.array(json_loads(x), dtype=numpy.float32)
+
+    return map(float, json_loads(x))
+
+
+def float32_vector_or_none(x: Optional[bytes]) -> Optional[Any]:
+    """
+    Covert value to float32 array.
+
+    Parameters
+    ----------
+    x : bytes or None
+        Little-endian block of bytes.
+
+    Returns
+    -------
+    float32 numpy array
+        If input value is not None and numpy is installed
+    float Python list
+        If input value is not None and numpy is not installed
+    None
+        If input value is None
+
+    """
+    if x is None:
+        return None
+
+    if has_numpy:
+        return numpy.frombuffer(x, dtype=numpy.float32)
+
+    return struct.unpack(f'<{len(x)/4}f', x)
+
+
+def float64_vector_json_or_none(x: Optional[str]) -> Optional[Any]:
+    """
+    Covert value to float64 array.
+
+    Parameters
+    ----------
+    x : str or None
+        JSON array
+
+    Returns
+    -------
+    float64 numpy array
+        If input value is not None and numpy is installed
+    float Python list
+        If input value is not None and numpy is not installed
+    None
+        If input value is None
+
+    """
+    if x is None:
+        return None
+
+    if has_numpy:
+        return numpy.array(json_loads(x), dtype=numpy.float64)
+
+    return map(float, json_loads(x))
+
+
+def float64_vector_or_none(x: Optional[bytes]) -> Optional[Any]:
+    """
+    Covert value to float64 array.
+
+    Parameters
+    ----------
+    x : bytes or None
+        JSON array
+
+    Returns
+    -------
+    float64 numpy array
+        If input value is not None and numpy is installed
+    float Python list
+        If input value is not None and numpy is not installed
+    None
+        If input value is None
+
+    """
+    if x is None:
+        return None
+
+    if has_numpy:
+        return numpy.frombuffer(x, dtype=numpy.float64)
+
+    return struct.unpack(f'<{len(x)/8}d', x)
+
+
+def int8_vector_json_or_none(x: Optional[str]) -> Optional[Any]:
+    """
+    Covert value to int8 array.
+
+    Parameters
+    ----------
+    x : str or None
+        JSON array
+
+    Returns
+    -------
+    int8 numpy array
+        If input value is not None and numpy is installed
+    int Python list
+        If input value is not None and numpy is not installed
+    None
+        If input value is None
+
+    """
+    if x is None:
+        return None
+
+    if has_numpy:
+        return numpy.array(json_loads(x), dtype=numpy.int8)
+
+    return map(int, json_loads(x))
+
+
+def int8_vector_or_none(x: Optional[bytes]) -> Optional[Any]:
+    """
+    Covert value to int8 array.
+
+    Parameters
+    ----------
+    x : bytes or None
+        Little-endian block of bytes.
+
+    Returns
+    -------
+    int8 numpy array
+        If input value is not None and numpy is installed
+    int Python list
+        If input value is not None and numpy is not installed
+    None
+        If input value is None
+
+    """
+    if x is None:
+        return None
+
+    if has_numpy:
+        return numpy.frombuffer(x, dtype=numpy.int8)
+
+    return struct.unpack(f'<{len(x)}b', x)
+
+
+def int16_vector_json_or_none(x: Optional[str]) -> Optional[Any]:
+    """
+    Covert value to int16 array.
+
+    Parameters
+    ----------
+    x : str or None
+        JSON array
+
+    Returns
+    -------
+    int16 numpy array
+        If input value is not None and numpy is installed
+    int Python list
+        If input value is not None and numpy is not installed
+    None
+        If input value is None
+
+    """
+    if x is None:
+        return None
+
+    if has_numpy:
+        return numpy.array(json_loads(x), dtype=numpy.int16)
+
+    return map(int, json_loads(x))
+
+
+def int16_vector_or_none(x: Optional[bytes]) -> Optional[Any]:
+    """
+    Covert value to int16 array.
+
+    Parameters
+    ----------
+    x : bytes or None
+        Little-endian block of bytes.
+
+    Returns
+    -------
+    int16 numpy array
+        If input value is not None and numpy is installed
+    int Python list
+        If input value is not None and numpy is not installed
+    None
+        If input value is None
+
+    """
+    if x is None:
+        return None
+
+    if has_numpy:
+        return numpy.frombuffer(x, dtype=numpy.int16)
+
+    return struct.unpack(f'<{len(x)/2}h', x)
+
+
+def int32_vector_json_or_none(x: Optional[str]) -> Optional[Any]:
+    """
+    Covert value to int32 array.
+
+    Parameters
+    ----------
+    x : str or None
+        JSON array
+
+    Returns
+    -------
+    int32 numpy array
+        If input value is not None and numpy is installed
+    int Python list
+        If input value is not None and numpy is not installed
+    None
+        If input value is None
+
+    """
+    if x is None:
+        return None
+
+    if has_numpy:
+        return numpy.array(json_loads(x), dtype=numpy.int32)
+
+    return map(int, json_loads(x))
+
+
+def int32_vector_or_none(x: Optional[bytes]) -> Optional[Any]:
+    """
+    Covert value to int32 array.
+
+    Parameters
+    ----------
+    x : bytes or None
+        Little-endian block of bytes.
+
+    Returns
+    -------
+    int32 numpy array
+        If input value is not None and numpy is installed
+    int Python list
+        If input value is not None and numpy is not installed
+    None
+        If input value is None
+
+    """
+    if x is None:
+        return None
+
+    if has_numpy:
+        return numpy.frombuffer(x, dtype=numpy.int32)
+
+    return struct.unpack(f'<{len(x)/4}l', x)
+
+
+def int64_vector_json_or_none(x: Optional[str]) -> Optional[Any]:
+    """
+    Covert value to int64 array.
+
+    Parameters
+    ----------
+    x : str or None
+        JSON array
+
+    Returns
+    -------
+    int64 numpy array
+        If input value is not None and numpy is installed
+    int Python list
+        If input value is not None and numpy is not installed
+    None
+        If input value is None
+
+    """
+    if x is None:
+        return None
+
+    if has_numpy:
+        return numpy.array(json_loads(x), dtype=numpy.int64)
+
+    return map(int, json_loads(x))
+
+
+def int64_vector_or_none(x: Optional[bytes]) -> Optional[Any]:
+    """
+    Covert value to int64 array.
+
+    Parameters
+    ----------
+    x : bytes or None
+        Little-endian block of bytes.
+
+    Returns
+    -------
+    int64 numpy array
+        If input value is not None and numpy is installed
+    int Python list
+        If input value is not None and numpy is not installed
+    None
+        If input value is None
+
+    """
+    if x is None:
+        return None
+
+    # Bytes
+    if has_numpy:
+        return numpy.frombuffer(x, dtype=numpy.int64)
+
+    return struct.unpack(f'<{len(x)/8}l', x)
+
+
+def bson_or_none(x: Optional[bytes]) -> Optional[Any]:
+    """
+    Convert a BSON value to a dictionary.
+
+    Parameters
+    ----------
+    x : bytes or None
+        BSON formatted bytes
+
+    Returns
+    -------
+    dict
+        If input value is not None and bson package is installed
+    bytes
+        If input value is not None and bson package is not installed
+    None
+        If input value is None
+
+    """
+    if x is None:
+        return None
+    if has_bson:
+        return bson.decode(x)
+    return x
+
+
 # Map of database types and conversion functions
 converters: Dict[int, Callable[..., Any]] = {
     0: decimal_or_none,
@@ -557,4 +934,17 @@ converters: Dict[int, Callable[..., Any]] = {
     #   253: identity,
     #   254: identity,
     255: geometry_or_none,
+    1001: bson_or_none,
+    2001: float32_vector_json_or_none,
+    2002: float64_vector_json_or_none,
+    2003: int8_vector_json_or_none,
+    2004: int16_vector_json_or_none,
+    2005: int32_vector_json_or_none,
+    2006: int64_vector_json_or_none,
+    3001: float32_vector_or_none,
+    3002: float64_vector_or_none,
+    3003: int8_vector_or_none,
+    3004: int16_vector_or_none,
+    3005: int32_vector_or_none,
+    3006: int64_vector_or_none,
 }

singlestoredb/functions/ext/asgi.py

@@ -455,10 +455,13 @@ class Application(object):
                         endpoints[name.encode('utf-8')] = func, info
 
                 # Fully qualified function name
-                else:
+                elif '.' in funcs:
                     pkg_path, func_names = funcs.rsplit('.', 1)
                     pkg = importlib.import_module(pkg_path)
 
+                    if pkg is None:
+                        raise RuntimeError(f'Could not locate module: {pkg}')
+
                     # Add endpoint for each exported function
                     for name, alias in get_func_names(func_names):
                         item = getattr(pkg, name)
@@ -466,6 +469,9 @@ class Application(object):
                         func, info = make_func(alias, item)
                         endpoints[alias.encode('utf-8')] = func, info
 
+                else:
+                    raise RuntimeError(f'Could not locate module: {funcs}')
+
             elif isinstance(funcs, ModuleType):
                 for x in vars(funcs).values():
                     if not hasattr(x, '_singlestoredb_attrs'):

singlestoredb/fusion/handler.py

@@ -220,6 +220,7 @@ def _format_arguments(arg: str) -> str:
 
 def _to_markdown(txt: str) -> str:
     """Convert formatting to markdown."""
+    txt = re.sub(r'`([^`]+)\s+\<([^\>]+)>`_', r'[\1](\2)', txt)
     txt = txt.replace('``', '`')
 
     # Format code blocks
@@ -228,11 +229,12 @@ def _to_markdown(txt: str) -> str:
     while lines:
         line = lines.pop(0)
         if line.endswith('::'):
-            out.append(line[:-2])
+            out.append(line[:-2] + '.')
             code = []
             while lines and (not lines[0].strip() or lines[0].startswith(' ')):
                 code.append(lines.pop(0).rstrip())
-            out.extend(['```sql', '\n'.join(code).rstrip(), '```\n'])
+            code_str = re.sub(r'^\s*\n', r'', '\n'.join(code).rstrip())
+            out.extend([f'```sql\n{code_str}\n```\n'])
         else:
             out.append(line)
 
@@ -241,9 +243,7 @@ def _to_markdown(txt: str) -> str:
 
 def build_help(syntax: str, grammar: str) -> str:
     """Build full help text."""
-    syntax = re.sub(r'\s*\n+\s*', r' ', syntax.strip())
-
-    cmd = re.match(r'([A-Z0-9_ ]+)', syntax)
+    cmd = re.match(r'([A-Z0-9_ ]+)', syntax.strip())
     if not cmd:
         raise ValueError(f'no command found: {syntax}')
 
@@ -264,7 +264,7 @@ def build_help(syntax: str, grammar: str) -> str:
     if 'description' in sections:
         out.extend([sections['description'], '\n\n'])
 
-    out.extend(['## Syntax\n```sql\n', syntax, '\n```\n\n'])
+    out.append(f'## Syntax\n\n```sql{syntax}\n```\n\n')
 
     if 'arguments' in sections:
         out.extend([
@@ -272,9 +272,15 @@ def build_help(syntax: str, grammar: str) -> str:
             _format_arguments(sections['arguments']),
             '\n\n',
         ])
+    if 'argument' in sections:
+        out.extend([
+            '## Argument\n\n',
+            _format_arguments(sections['argument']),
+            '\n\n',
+        ])
 
     if 'remarks' in sections:
-        out.extend(['## Remarks\n', sections['remarks'], '\n\n'])
+        out.extend(['## Remarks\n\n', sections['remarks'], '\n\n'])
 
     if 'examples' in sections:
         out.extend(['## Examples\n\n', _format_examples(sections['examples']), '\n\n'])
@@ -282,7 +288,7 @@ def build_help(syntax: str, grammar: str) -> str:
         out.extend(['## Example\n\n', _format_examples(sections['example']), '\n\n'])
 
     if 'see also' in sections:
-        out.extend(['## See Also\n', sections['see also'], '\n\n'])
+        out.extend(['## See Also\n\n', sections['see also'], '\n\n'])
 
     return ''.join(out).rstrip() + '\n'