singlestoredb 0.8.8__cp36-abi3-win_amd64.whl → 0.9.0__cp36-abi3-win_amd64.whl

singlestoredb/functions/ext/__init__.py

@@ -0,0 +1,2 @@
+#!/usr/bin/env python3
+from .asgi import create_app  # noqa: F401

singlestoredb/functions/ext/asgi.py

@@ -0,0 +1,357 @@
+#!/usr/bin/env python3
+'''
+Web application for SingleStoreDB external functions.
+
+This module supplies a function that can create web apps intended for use
+with the external function feature of SingleStoreDB. The application
+function is a standard ASGI <https://asgi.readthedocs.io/en/latest/index.html>
+request handler for use with servers such as Uvicorn <https://www.uvicorn.org>.
+
+An external function web application can be created using the `create_app`
+function. By default, the exported Python functions are specified by
+environment variables starting with SINGLESTOREDB_EXT_FUNCTIONS. See the
+documentation in `create_app` for the full syntax. If the application is
+created in Python code rather than from the command-line, exported
+functions can be specified in the parameters.
+
+An example of starting a server is shown below.
+
+Example
+-------
+$ SINGLESTOREDB_EXT_FUNCTIONS='myfuncs.[percentage_90,percentage_95]' \
+    uvicorn --factory singlestoredb.functions.ext:create_app
+
+'''
+import importlib
+import itertools
+import os
+from typing import Any
+from typing import Awaitable
+from typing import Callable
+from typing import Dict
+from typing import Iterable
+from typing import List
+from typing import Optional
+from typing import Sequence
+from typing import Tuple
+from typing import Union
+
+from . import json as jdata
+from . import rowdat_1
+from ...mysql.constants import FIELD_TYPE as ft
+from ..signature import get_signature
+from ..signature import signature_to_sql
+
+# If a number of processes is specified, create a pool of workers
+num_processes = max(0, int(os.environ.get('SINGLESTOREDB_EXT_NUM_PROCESSES', 0)))
+if num_processes > 1:
+    try:
+        from ray.util.multiprocessing import Pool
+    except ImportError:
+        from multiprocessing import Pool
+    func_map = Pool(num_processes).starmap
+else:
+    func_map = itertools.starmap
+
+
+# Use negative values to indicate unsigned ints / binary data / usec time precision
+rowdat_1_type_map = {
+    'bool': ft.LONGLONG,
+    'int8': ft.LONGLONG,
+    'int16': ft.LONGLONG,
+    'int32': ft.LONGLONG,
+    'int64': ft.LONGLONG,
+    'uint8': -ft.LONGLONG,
+    'uint16': -ft.LONGLONG,
+    'uint32': -ft.LONGLONG,
+    'uint64': -ft.LONGLONG,
+    'float32': ft.DOUBLE,
+    'float64': ft.DOUBLE,
+    'str': ft.STRING,
+    'bytes': -ft.STRING,
+}
+
+
+def get_func_names(funcs: str) -> List[Tuple[str, str]]:
+    '''
+    Parse all function names from string.
+
+    Parameters
+    ----------
+    func_names : str
+        String containing one or more function names. The syntax is
+        as follows: [func-name-1@func-alias-1,func-name-2@func-alias-2,...].
+        The optional '@name' portion is an alias if you want the function
+        to be renamed.
+
+    Returns
+    -------
+    List[Tuple[str]] : a list of tuples containing the names and aliases
+        of each function.
+
+    '''
+    if funcs.startswith('['):
+        func_names = funcs.replace('[', '').replace(']', '').split(',')
+        func_names = [x.strip() for x in func_names]
+    else:
+        func_names = [funcs]
+
+    out = []
+    for name in func_names:
+        alias = name
+        if '@' in name:
+            name, alias = name.split('@', 1)
+        out.append((name, alias))
+
+    return out
+
+
+def make_func(name: str, func: Callable[..., Any]) -> Callable[..., Any]:
+    '''
+    Make a function endpoint.
+
+    Parameters
+    ----------
+    name : str
+        Name of the function to create
+    func : Callable
+        The function to call as the endpoint
+
+    Returns
+    -------
+    Callable
+
+    '''
+    async def do_func(row_ids: Sequence[int], rows: Sequence[Any]) -> List[Any]:
+        '''Call function on given rows of data.'''
+        return list(zip(row_ids, func_map(func, rows)))
+
+    do_func.__name__ = name
+    do_func.__doc__ = func.__doc__
+
+    sig = get_signature(func, name=name)
+
+    # Store signature for generating CREATE FUNCTION calls
+    do_func._ext_func_signature = sig  # type: ignore
+
+    # Setup argument types for rowdat_1 parser
+    colspec = []
+    for x in sig['args']:
+        dtype = x['dtype'].replace('?', '')
+        if dtype not in rowdat_1_type_map:
+            raise TypeError(f'no data type mapping for {dtype}')
+        colspec.append((x['name'], rowdat_1_type_map[dtype]))
+    do_func._ext_func_colspec = colspec  # type: ignore
+
+    # Setup return type
+    dtype = sig['returns']['dtype'].replace('?', '')
+    if dtype not in rowdat_1_type_map:
+        raise TypeError(f'no data type mapping for {dtype}')
+    do_func._ext_func_returns = [rowdat_1_type_map[dtype]]  # type: ignore
+
+    return do_func
+
+
+def create_app(
+    functions: Optional[
+        Union[
+            str,
+            Iterable[str],
+            Callable[..., Any],
+            Iterable[Callable[..., Any]],
+        ]
+    ] = None,
+
+
+) -> Callable[..., Any]:
+    '''
+    Create an external function application.
+
+    If `functions` is None, the environment is searched for function
+    specifications in variables starting with `SINGLESTOREDB_EXT_FUNCTIONS`.
+    Any number of environment variables can be specified as long as they
+    have this prefix. The format of the environment variable value is the
+    same as for the `functions` parameter.
+
+    Parameters
+    ----------
+    functions : str or Iterable[str], optional
+        Python functions are specified using a string format as follows:
+            * Single function : <pkg1>.<func1>
+            * Multiple functions : <pkg1>.[<func1-name,func2-name,...]
+            * Function aliases : <pkg1>.[<func1@alias1,func2@alias2,...]
+            * Multiple packages : <pkg1>.<func1>:<pkg2>.<func2>
+
+    Returns
+    -------
+    Callable : the application request handler
+
+    '''
+
+    # List of functions specs
+    specs: List[Union[str, Callable[..., Any]]] = []
+
+    # Look up Python function specifications
+    if functions is None:
+        for k, v in os.environ.items():
+            if k.startswith('SINGLESTOREDB_EXT_FUNCTIONS'):
+                specs.append(v)
+    elif isinstance(functions, str):
+        specs = [functions]
+    elif callable(functions):
+        specs = [functions]
+    else:
+        specs = list(functions)
+
+    # Add functions to application
+    endpoints = dict()
+    for funcs in itertools.chain(specs):
+        if isinstance(funcs, str):
+            pkg_path, func_names = funcs.rsplit('.', 1)
+            pkg = importlib.import_module(pkg_path)
+
+            # Add endpoint for each exported function
+            for name, alias in get_func_names(func_names):
+                item = getattr(pkg, name)
+                func = make_func(alias, item)
+                endpoints[alias.encode('utf-8')] = func
+        else:
+            alias = funcs.__name__
+            func = make_func(alias, item)
+            endpoints[alias.encode('utf-8')] = func
+
+    # Plain text response start
+    text_response_dict: Dict[str, Any] = dict(
+        type='http.response.start',
+        status=200,
+        headers=[(b'content-type', b'text/plain')],
+    )
+
+    # JSON response start
+    json_response_dict: Dict[str, Any] = dict(
+        type='http.response.start',
+        status=200,
+        headers=[(b'content-type', b'application/json')],
+    )
+
+    # ROWDAT_1 response start
+    rowdat_1_response_dict: Dict[str, Any] = dict(
+        type='http.response.start',
+        status=200,
+        headers=[(b'content-type', b'x-application/rowdat_1')],
+    )
+
+    # Path not found response start
+    path_not_found_response_dict: Dict[str, Any] = dict(
+        type='http.response.start',
+        status=404,
+    )
+
+    # Response body template
+    body_response_dict: Dict[str, Any] = dict(
+        type='http.response.body',
+    )
+
+    # Data format + version handlers
+    handlers = {
+        (b'application/octet-stream', b'1.0'): dict(
+            load=rowdat_1.load,
+            dump=rowdat_1.dump,
+            response=rowdat_1_response_dict,
+        ),
+        (b'application/json', b'1.0'): dict(
+            load=jdata.load,
+            dump=jdata.dump,
+            response=json_response_dict,
+        ),
+    }
+
+    # Valid URL paths
+    invoke_path = ('invoke',)
+    show_create_function_path = ('show', 'create_function')
+
+    async def app(
+        scope: Dict[str, Any],
+        receive: Callable[..., Awaitable[Any]],
+        send: Callable[..., Awaitable[Any]],
+    ) -> None:
+        '''
+        Application request handler.
+
+        Parameters
+        ----------
+        scope : dict
+            ASGI request scope
+        receive : Callable
+            Function to receieve request information
+        send : Callable
+            Function to send response information
+
+        '''
+        assert scope['type'] == 'http'
+
+        method = scope['method']
+        path = tuple(x for x in scope['path'].split('/') if x)
+        headers = dict(scope['headers'])
+
+        content_type = headers.get(
+            b'content-type',
+            b'application/octet-stream',
+        )
+        accepts = headers.get(b'accepts', content_type)
+        func_name = headers.get(b's2-ef-name', b'')
+        func = endpoints.get(func_name)
+
+        # Call the endpoint
+        if method == 'POST' and func is not None and path == invoke_path:
+            data = []
+            more_body = True
+            while more_body:
+                request = await receive()
+                data.append(request['body'])
+                more_body = request.get('more_body', False)
+
+            data_version = headers.get(b's2-ef-version', b'')
+            input_handler = handlers[(content_type, data_version)]
+            output_handler = handlers[(accepts, data_version)]
+
+            out = await func(
+                *input_handler['load'](
+                    func._ext_func_colspec, b''.join(data),  # type: ignore
+                ),
+            )
+            body = output_handler['dump'](func._ext_func_returns, out)  # type: ignore
+
+            await send(output_handler['response'])
+
+        # Handle api reflection
+        elif method == 'GET' and path == show_create_function_path:
+            host = headers.get(b'host', b'localhost:80')
+            url = f'{scope["scheme"]}://{host.decode("utf-8")}/invoke'
+            data_format = 'json' if b'json' in content_type else 'rowdat_1'
+
+            syntax = []
+            for key, endpoint in endpoints.items():
+                if not func_name or key == func_name:
+                    syntax.append(
+                        signature_to_sql(
+                            endpoint._ext_func_signature,  # type: ignore
+                            base_url=url,
+                            data_format=data_format,
+                        ),
+                    )
+            body = '\n'.join(syntax).encode('utf-8')
+
+            await send(text_response_dict)
+
+        # Path not found
+        else:
+            body = b''
+            await send(path_not_found_response_dict)
+
+        # Send body
+        out = body_response_dict.copy()
+        out['body'] = body
+        await send(out)
+
+    return app

singlestoredb/functions/ext/json.py

@@ -0,0 +1,49 @@
+#!/usr/bin/env python3
+import json
+from typing import Any
+from typing import Iterable
+from typing import List
+from typing import Tuple
+
+
+def load(colspec: Iterable[Tuple[str, int]], data: bytes) -> Tuple[List[int], List[Any]]:
+    '''
+    Convert bytes in JSON format into rows of data.
+
+    Parameters
+    ----------
+    colspec : Iterable[str]
+        An Iterable of column data types
+    data : bytes
+        The data in JSON format
+
+    Returns
+    -------
+    list[list[Any]]
+
+    '''
+    row_ids = []
+    rows = []
+    for row_id, *row in json.loads(data.decode('utf-8'))['data']:
+        row_ids.append(row_id)
+        rows.append(row)
+    return row_ids, rows
+
+
+def dump(returns: Iterable[int], data: Iterable[Tuple[int, Any]]) -> bytes:
+    '''
+    Convert a list of lists of data into JSON format.
+
+    Parameters
+    ----------
+    returns : str
+        The returned data type
+    data : list[list[Any]]
+        The rows of data to serialize
+
+    Returns
+    -------
+    bytes
+
+    '''
+    return json.dumps(dict(data=list(data))).encode('utf-8')

singlestoredb/functions/ext/rowdat_1.py

@@ -0,0 +1,111 @@
+#!/usr/bin/env python3
+import struct
+import warnings
+from io import BytesIO
+from typing import Any
+from typing import Iterable
+from typing import List
+from typing import Tuple
+
+try:
+    import _singlestoredb_accel
+except ImportError:
+    warnings.warn(
+        'could not load accelerated data reader for external functions; '
+        'using pure Python implementation.',
+        RuntimeWarning,
+    )
+    _singlestoredb_accel = None
+
+from ...mysql.constants import FIELD_TYPE as ft
+
+
+def _load(colspec: Iterable[Tuple[str, int]], data: bytes) -> Tuple[List[int], List[Any]]:
+    '''
+    Convert bytes in rowdat_1 format into rows of data.
+
+    Parameters
+    ----------
+    colspec : Iterable[str]
+        An Iterable of column data types
+    data : bytes
+        The data in rowdat_1 format
+
+    Returns
+    -------
+    list[list[Any]]
+
+    '''
+    data_len = len(data)
+    data_io = BytesIO(data)
+    row_ids = []
+    rows = []
+    val = None
+    while data_io.tell() < data_len:
+        row_ids.append(struct.unpack('<q', data_io.read(8))[0])
+        row = []
+        for _, ctype in colspec:
+            is_null = data_io.read(1) == b'\x01'
+            if ctype is ft.LONGLONG:
+                val = struct.unpack('<q', data_io.read(8))[0]
+            elif ctype is ft.DOUBLE:
+                val = struct.unpack('<d', data_io.read(8))[0]
+            elif ctype is ft.STRING:
+                slen = struct.unpack('<q', data_io.read(8))[0]
+                val = data_io.read(slen).decode('utf-8')
+            else:
+                raise TypeError(f'unrecognized column type: {ctype}')
+            row.append(None if is_null else val)
+        rows.append(row)
+    return row_ids, rows
+
+
+def _dump(returns: Iterable[int], data: Iterable[Tuple[int, Any]]) -> bytes:
+    '''
+    Convert a list of lists of data into rowdat_1 format.
+
+    Parameters
+    ----------
+    returns : str
+        sThe returned data type
+    data : list[list[Any]]
+        The rows of data to serialize
+
+    Returns
+    -------
+    bytes
+
+    '''
+    out = BytesIO()
+    for row_id, value in data:
+        out.write(struct.pack('<q', row_id))
+        for rtype in returns:
+            out.write(b'\x01' if value is None else b'\x00')
+            if rtype is ft.LONGLONG:
+                if value is None:
+                    out.write(struct.pack('<q', 0))
+                else:
+                    out.write(struct.pack('<q', value))
+            elif rtype is ft.DOUBLE:
+                if value is None:
+                    out.write(struct.pack('<d', 0.0))
+                else:
+                    out.write(struct.pack('<d', value))
+            elif rtype is ft.STRING:
+                if value is None:
+                    out.write(struct.pack('<q', 0))
+                else:
+                    sval = value.encode('utf-8')
+                    out.write(struct.pack('<q', len(sval)))
+                    out.write(sval)
+            else:
+                raise TypeError(f'unrecognized column type: {rtype}')
+    return out.getvalue()
+
+
+if _singlestoredb_accel is None:
+    load = _load
+    dump = _dump
+else:
+    load = _singlestoredb_accel.load_rowdat_1
+    dump = _singlestoredb_accel.dump_rowdat_1