singlestoredb 1.0.1__cp38-abi3-win_amd64.whl → 1.0.3__cp38-abi3-win_amd64.whl

singlestoredb/__init__.py

@@ -13,7 +13,7 @@ Examples
 
 """
 
-__version__ = '1.0.1'
+__version__ = '1.0.3'
 
 from typing import Any
 

singlestoredb/config.py

@@ -253,6 +253,18 @@ register_option(
     environ=['SINGLESTOREDB_MANAGEMENT_TOKEN'],
 )
 
+register_option(
+    'management.base_url', 'string', check_str, 'https://api.singlestore.com',
+    'Specifies the base URL for the management API.',
+    environ=['SINGLESTOREDB_MANAGEMENT_BASE_URL'],
+)
+
+register_option(
+    'management.version', 'string', check_str, 'v1',
+    'Specifies the version for the management API.',
+    environ=['SINGLESTOREDB_MANAGEMENT_VERSION'],
+)
+
 
 #
 # Debugging options

singlestoredb/functions/decorator.py

@@ -22,18 +22,8 @@ def udf(
     func: Optional[Callable[..., Any]] = None,
     *,
     name: Optional[str] = None,
-    database: Optional[str] = None,
-    environment: Optional[str] = None,
-    packages: Optional[Union[str, List[str]]] = None,
-    resources: Optional[Union[str, List[str]]] = None,
-    max_batch_size: int = 500,
-    n_processes: int = 1,
-    n_instances: int = 1,
     args: Optional[Union[DataType, List[DataType], Dict[str, DataType]]] = None,
     returns: Optional[str] = None,
-    replace: bool = False,
-    remote_service: Optional[str] = None,
-    link: Optional[str] = None,
     data_format: Optional[str] = None,
     include_masks: bool = False,
 ) -> Callable[..., Any]:
@@ -46,27 +36,6 @@ def udf(
         The UDF to apply parameters to
     name : str, optional
         The name to use for the UDF in the database
-    database : str, optional
-        The database to create the functions in
-    environment : str, optional
-        The environment to create for the functions
-    packages : str or list-of-strs, optional
-        The package dependency specifications. Strings must be in the
-        format used in requirements.txt files.
-    max_match_size : int, optional
-        The number of rows to batch in the server before sending
-        them to the UDF application
-    n_processes : int, optional
-        The number of sub-processes to spin up to process sub-batches
-        of rows of data. This may be used if the UDF is CPU-intensive
-        and there are free CPUs in the server the web application
-        is running in. If the UDF is very short-running, setting this
-        parameter to greater than one will likey cause the UDF to
-        run slower since the overhead of the extra processes and moving
-        data would be the limiting factor.
-    n_instances : int, optional
-        The number of runtime environments to use to handle data
-        processing requests
     args : str | Callable | List[str | Callable] | Dict[str, str | Callable], optional
         Specifies the data types of the function arguments. Typically,
         the function data types are derived from the function parameter
@@ -83,14 +52,6 @@ def udf(
     returns : str, optional
         Specifies the return data type of the function. If not specified,
         the type annotation from the function is used.
-    replace : bool, optional
-        Should an existing function of the same name be replaced when
-        creating the function in the database?
-    remote_service : str, optional
-        URL of the remote service that handles this function. If using an
-        environment, this URL is generated automatically.
-    link : str, optional
-        Name of link to use to connect to remote service
     data_format : str, optional
         The data format of each parameter: python, pandas, arrow, polars
     include_masks : bool, optional
@@ -103,10 +64,6 @@ def udf(
     Callable
 
     """
-    assert max_batch_size >= 1
-    assert n_processes >= 1
-    assert n_instances >= 1
-
     if args is None:
         pass
     elif isinstance(args, (list, tuple)):
@@ -153,18 +110,8 @@ def udf(
     _singlestoredb_attrs = {  # type: ignore
         k: v for k, v in dict(
             name=name,
-            database=database,
-            environment=environment,
-            packages=listify(packages),
-            resources=listify(resources),
-            max_batch_size=max(1, int(max_batch_size)),
-            n_processes=max(1, int(n_processes)),
-            n_instances=max(1, int(n_instances)),
             args=args,
             returns=returns,
-            replace=bool(replace),
-            remote_service=remote_service,
-            link=link,
             data_format=data_format,
             include_masks=include_masks,
         ).items() if v is not None

singlestoredb/functions/ext/asgi.py

@@ -25,8 +25,10 @@ $ SINGLESTOREDB_EXT_FUNCTIONS='myfuncs.[percentage_90,percentage_95]' \
 import importlib.util
 import io
 import itertools
+import json
 import os
-import urllib
+import re
+import secrets
 from types import ModuleType
 from typing import Any
 from typing import Awaitable
@@ -36,6 +38,7 @@ from typing import Iterable
 from typing import List
 from typing import Optional
 from typing import Sequence
+from typing import Set
 from typing import Tuple
 from typing import Union
 
@@ -196,8 +199,12 @@ def create_app(  # noqa: C901
             Iterable[ModuleType],
         ]
     ] = None,
-
-
+    app_mode: str = 'remote',
+    url: str = 'http://localhost:8000/invoke',
+    data_format: str = 'rowdat_1',
+    data_version: str = '1.0',
+    link_config: Optional[Dict[str, Any]] = None,
+    link_credentials: Optional[Dict[str, Any]] = None,
 ) -> Callable[..., Any]:
     '''
     Create an external function application.
@@ -216,6 +223,20 @@ def create_app(  # noqa: C901
             * Multiple functions : <pkg1>.[<func1-name,func2-name,...]
             * Function aliases : <pkg1>.[<func1@alias1,func2@alias2,...]
             * Multiple packages : <pkg1>.<func1>:<pkg2>.<func2>
+    app_mode : str, optional
+        The mode of operation for the application: remote or collocated
+    url : str, optional
+        The URL of the function API
+    data_format : str, optional
+        The format of the data rows: 'rowdat_1' or 'json'
+    data_version : str, optional
+        The version of the call format to expect: '1.0'
+    link_config : Dict[str, Any], optional
+        The CONFIG section of a LINK definition. This dictionary gets
+        converted to JSON for the CREATE LINK call.
+    link_credentials : Dict[str, Any], optional
+        The CREDENTIALS section of a LINK definition. This dictionary gets
+        converted to JSON for the CREATE LINK call.
 
     Returns
     -------
@@ -469,7 +490,7 @@ def create_app(  # noqa: C901
         # Handle api reflection
         elif method == 'GET' and path == show_create_function_path:
             host = headers.get(b'host', b'localhost:80')
-            url = f'{scope["scheme"]}://{host.decode("utf-8")}/invoke'
+            reflected_url = f'{scope["scheme"]}://{host.decode("utf-8")}/invoke'
             data_format = 'json' if b'json' in content_type else 'rowdat_1'
 
             syntax = []
@@ -478,7 +499,7 @@ def create_app(  # noqa: C901
                     syntax.append(
                         signature_to_sql(
                             endpoint._ext_func_signature,  # type: ignore
-                            base_url=url,
+                            url=url or reflected_url,
                             data_format=data_format,
                         ),
                     )
@@ -496,35 +517,88 @@ def create_app(  # noqa: C901
         out['body'] = body
         await send(out)
 
+    def _create_link(
+        config: Optional[Dict[str, Any]],
+        credentials: Optional[Dict[str, Any]],
+    ) -> Tuple[str, str]:
+        """Generate CREATE LINK command."""
+        if not config and not credentials:
+            return '', ''
+
+        link_name = f'link_{secrets.token_hex(16)}'
+        out = [f'CREATE LINK {link_name} AS HTTP']
+
+        if config:
+            out.append(f"CONFIG '{json.dumps(config)}'")
+
+        if credentials:
+            out.append(f"CREDENTIALS '{json.dumps(credentials)}'")
+
+        return link_name, ' '.join(out) + ';'
+
+    def _locate_app_functions(cur: Any) -> Tuple[Set[str], Set[str]]:
+        """Locate all current functions and links belonging to this app."""
+        funcs, links = set(), set()
+        cur.execute('SHOW FUNCTIONS')
+        for name, ftype, _, _, _, link in list(cur):
+            # Only look at external functions
+            if 'external' not in ftype.lower():
+                continue
+            # See if function URL matches url
+            cur.execute(f'SHOW CREATE FUNCTION `{name}`')
+            for fname, _, code, *_ in list(cur):
+                m = re.search(r" (?:\w+) SERVICE '([^']+)'", code)
+                if m and m.group(1) == url:
+                    funcs.add(fname)
+                    if link:
+                        links.add(link)
+        return funcs, links
+
     def show_create_functions(
-        base_url: str = 'http://localhost:8000',
-        data_format: str = 'rowdat_1',
+        replace: bool = False,
     ) -> List[str]:
+        """Generate CREATE FUNCTION calls."""
+        if not endpoints:
+            return []
+
         out = []
+        link = ''
+        if app_mode.lower() == 'remote':
+            link, link_str = _create_link(link_config, link_credentials)
+            if link and link_str:
+                out.append(link_str)
+
         for key, endpoint in endpoints.items():
             out.append(
                 signature_to_sql(
                     endpoint._ext_func_signature,  # type: ignore
-                    base_url=urllib.parse.urljoin(base_url, '/invoke'),
+                    url=url,
                     data_format=data_format,
+                    app_mode=app_mode,
+                    replace=replace,
+                    link=link or None,
                 ),
             )
+
         return out
 
     app.show_create_functions = show_create_functions  # type: ignore
 
     def register_functions(
         *connection_args: Any,
-        base_url: str = 'http://localhost:8000',
-        data_format: str = 'rowdat_1',
+        replace: bool = False,
         **connection_kwargs: Any,
     ) -> None:
+        """Register functions with the database."""
         with connection.connect(*connection_args, **connection_kwargs) as conn:
             with conn.cursor() as cur:
-                for func in app.show_create_functions(  # type: ignore
-                                base_url=base_url,
-                                data_format=data_format,
-                ):  # type: ignore
+                if replace:
+                    funcs, links = _locate_app_functions(cur)
+                    for fname in funcs:
+                        cur.execute(f'DROP FUNCTION IF EXISTS `{fname}`')
+                    for link in links:
+                        cur.execute(f'DROP LINK {link}')
+                for func in app.show_create_functions(replace=replace):  # type: ignore
                     cur.execute(func)
 
     app.register_functions = register_functions  # type: ignore
@@ -533,10 +607,14 @@ def create_app(  # noqa: C901
         *connection_args: Any,
         **connection_kwargs: Any,
     ) -> None:
+        """Drop registered functions from database."""
         with connection.connect(*connection_args, **connection_kwargs) as conn:
             with conn.cursor() as cur:
-                for key in endpoints.keys():
-                    cur.execute(f'DROP FUNCTION IF EXISTS `{key.decode("utf8")}`')
+                funcs, links = _locate_app_functions(cur)
+                for fname in funcs:
+                    cur.execute(f'DROP FUNCTION IF EXISTS `{fname}`')
+                for link in links:
+                    cur.execute(f'DROP LINK {link}')
 
     app.drop_functions = drop_functions  # type: ignore
 
@@ -544,8 +622,8 @@ def create_app(  # noqa: C901
         name: str,
         data_in: io.BytesIO,
         data_out: io.BytesIO,
-        data_format: str = 'rowdat_1',
-        data_version: str = '1.0',
+        data_format: str = data_format,
+        data_version: str = data_version,
     ) -> None:
 
         async def receive() -> Dict[str, Any]:

singlestoredb/functions/ext/mmap.py

@@ -0,0 +1,306 @@
+#!/usr/bin/env python
+'''
+Module for creating collocated Python UDFs
+
+This module implements the collocated form of external functions for
+SingleStoreDB. This mode uses a socket for control communications
+and memory mapped files for passing the data to and from the UDF.
+
+The command below is a sample invocation. It exports all functions
+within the `myfuncs` Python module that have a `@udf` decorator on
+them. The `--db` option specifies a database connection string.
+If this exists, the UDF application will connect to the database
+and register all functions. The `--replace-existing` option indicates
+that existing functions should be replaced::
+
+    python -m singlestoredb.functions.ext.mmap \
+        --db=root:@127.0.0.1:9306/cosmeticshop --replace-existing \
+        myfuncs
+
+The `myfuncs` package can be any Python package in your Python path.
+It must contain functions marked with a `@udf` decorator and the
+types must be annotated or specified using the `@udf` decorator
+similar to the following::
+
+    from singlestoredb.functions import udf
+
+    @udf
+    def print_it(x2: float, x3: str) -> str:
+        return int(x2) * x3
+
+    @udf.pandas
+    def print_it_pandas(x2: float, x3: str) -> str:
+        return x2.astype(np.int64) * x3.astype(str)
+
+With the functions registered, you can now run the UDFs::
+
+    SELECT print_it(3.14, 'my string');
+    SELECT print_it_pandas(3.14, 'my string');
+
+'''
+import argparse
+import array
+import asyncio
+import io
+import logging
+import mmap
+import multiprocessing
+import os
+import secrets
+import socket
+import struct
+import sys
+import tempfile
+import threading
+import traceback
+from typing import Any
+
+from . import asgi
+
+
+logger = logging.getLogger('singlestoredb.functions.ext.mmap')
+handler = logging.StreamHandler()
+formatter = logging.Formatter('%(asctime)s %(name)-12s %(levelname)-8s %(message)s')
+handler.setFormatter(formatter)
+logger.addHandler(handler)
+logger.setLevel(logging.INFO)
+
+
+def _handle_request(app: Any, connection: Any, client_address: Any) -> None:
+    '''
+    Handle function call request.
+
+    Parameters:
+    app : ASGI app
+        An ASGI application from the singlestoredb.functions.ext.asgi module
+    connection : socket connection
+        Socket connection for function control messages
+    client_address : string
+        Address of connecting client
+
+    '''
+    logger.info('connection from {}'.format(str(connection).split(', ')[0][-4:]))
+
+    # Receive the request header.  Format:
+    #   server version:          uint64
+    #   length of function name: uint64
+    buf = connection.recv(16)
+    version, namelen = struct.unpack('<qq', buf)
+
+    # Python's recvmsg returns a tuple.  We only really care about the first
+    # two parts.  The recvmsg call has a weird way of specifying the size for
+    # the file descriptor array; basically, we're indicating we want to read
+    # two 32-bit ints (for the input and output files).
+    fd_model = array.array('i', [0, 0])
+    msg, ancdata, flags, addr = connection.recvmsg(
+        namelen,
+        socket.CMSG_LEN(2 * fd_model.itemsize),
+    )
+    assert len(ancdata) == 1
+
+    # The function's name will be in the "message" area of the recvmsg response.
+    # It will be populated with `namelen` bytes.
+    name = msg.decode('utf8')
+
+    # Two file descriptors are transferred to us from the database via the
+    # `sendmsg` protocol.  These are for reading the input rows and writing
+    # the output rows, respectively.
+    fd0, fd1 = struct.unpack('<ii', ancdata[0][2])
+    ifile = os.fdopen(fd0, 'rb')
+    ofile = os.fdopen(fd1, 'wb')
+
+    # Keep receiving data on this socket until we run out.
+    while True:
+
+        # Read in the length of this row, a uint64.  No data means we're done
+        # receiving.
+        buf = connection.recv(8)
+        if not buf:
+            break
+        length = struct.unpack('<q', buf)[0]
+        if not length:
+            break
+
+        # Map in the input shared memory segment from the fd we received via
+        # recvmsg.
+        mem = mmap.mmap(
+            ifile.fileno(),
+            length,
+            mmap.MAP_SHARED,
+            mmap.PROT_READ,
+        )
+
+        # Read row data
+        response_size = 0
+        out = io.BytesIO()
+
+        ifile.seek(0)
+        try:
+            # Run the function
+            asyncio.run(
+                app.call(
+                    name,
+                    io.BytesIO(ifile.read(length)),
+                    out,
+                    data_format='rowdat_1',
+                    data_version='1.0',
+                ),
+            )
+
+            # Write results
+            buf = out.getbuffer()
+            response_size = len(buf)
+            ofile.truncate(max(128*1024, response_size))
+            ofile.seek(0)
+            ofile.write(buf)
+            ofile.flush()
+
+            # Complete the request by send back the status as two uint64s on the
+            # socket:
+            #     - http status
+            #     - size of data in output shared memory
+            connection.send(struct.pack('<qq', 200, response_size))
+
+        except Exception as exc:
+            errmsg = f'error occurred in executing function `{name}`: {exc}\n'
+            logger.error(errmsg.rstrip())
+            for line in traceback.format_exception(exc):  # type: ignore
+                logger.error(line.rstrip())
+            connection.send(
+                struct.pack(
+                    f'<qq{len(errmsg)}s', 500,
+                    len(errmsg), errmsg.encode('utf8'),
+                ),
+            )
+            break
+
+        finally:
+            # Close the shared memory object.
+            mem.close()
+
+    # Close shared memory files.
+    ifile.close()
+    ofile.close()
+
+    # Close the connection
+    connection.close()
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(
+        prog='python -m singlestoredb.functions.ext.mmap',
+        description='Run a collacated Python UDF server',
+    )
+    parser.add_argument(
+        '--max-connections', metavar='n', type=int, default=32,
+        help='maximum number of server connections before refusing them',
+    )
+    parser.add_argument(
+        '--single-thread', default=False, action='store_true',
+        help='should the server run in single-thread mode?',
+    )
+    parser.add_argument(
+        '--socket-path', metavar='file-path',
+        default=os.path.join(tempfile.gettempdir(), secrets.token_hex(16)),
+        help='path to communications socket',
+    )
+    parser.add_argument(
+        '--db', metavar='conn-str', default='',
+        help='connection string to use for registering functions',
+    )
+    parser.add_argument(
+        '--replace-existing', action='store_true',
+        help='should existing functions of the same name '
+             'in the database be replaced?',
+    )
+    parser.add_argument(
+        '--log-level', metavar='[info|debug|warning|error]', default='info',
+        help='logging level',
+    )
+    parser.add_argument(
+        '--process-mode', metavar='[thread|subprocess]', default='subprocess',
+        help='how to handle concurrent handlers',
+    )
+    parser.add_argument(
+        'functions', metavar='module.or.func.path', nargs='*',
+        help='functions or modules to export in UDF server',
+    )
+    args = parser.parse_args()
+
+    logger.setLevel(getattr(logging, args.log_level.upper()))
+
+    if os.path.exists(args.socket_path):
+        try:
+            os.unlink(args.socket_path)
+        except (IOError, OSError):
+            logger.error(f'could not remove existing socket path: {args.socket_path}')
+            sys.exit(1)
+
+    # Create application
+    app = asgi.create_app(
+        args.functions,
+        app_mode='collocated',
+        data_format='rowdat_1',
+        url=args.socket_path,
+    )
+
+    funcs = app.show_create_functions(replace=True)  # type: ignore
+    if not funcs:
+        logger.error('no functions specified')
+        sys.exit(1)
+
+    for f in funcs:
+        logger.info(f'function: {f}')
+
+    # Register functions with database
+    if args.db:
+        logger.info('registering functions with database')
+        app.register_functions(args.db, replace=args.replace_existing)  # type: ignore
+
+    # Create the Unix socket server.
+    server = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+
+    # Bind our server to the path.
+    server.bind(args.socket_path)
+
+    logger.info(f'using socket path: {args.socket_path}')
+
+    # Listen for incoming connections. Argument is the number of connections to
+    # keep in the backlog before we begin refusing them; 32 is plenty for this
+    # simple case.
+    server.listen(args.max_connections)
+
+    # Accept connections forever.
+    try:
+        while True:
+            # Listen for the next connection on our port.
+            connection, client_address = server.accept()
+
+            if args.process_mode == 'thread':
+                tcls = threading.Thread
+            else:
+                tcls = multiprocessing.Process  # type: ignore
+
+            t = tcls(
+                target=_handle_request,
+                args=(app, connection, client_address),
+            )
+
+            t.start()
+
+            # NOTE: The following line forces this process to handle requests
+            # serially. This makes it easier to understand what's going on.
+            # In real life, though, parallel is much faster. To use parallel
+            # handling, just comment out the next line.
+            if args.single_thread:
+                t.join()
+
+    except KeyboardInterrupt:
+        sys.exit(0)
+
+    finally:
+        # Remove the socket file before we exit.
+        try:
+            os.unlink(args.socket_path)
+        except (IOError, OSError):
+            logger.error(f'could not remove socket path: {args.socket_path}')

singlestoredb/functions/signature.py

@@ -5,7 +5,6 @@ import numbers
 import os
 import re
 import string
-import textwrap
 import typing
 from typing import Any
 from typing import Callable
@@ -16,7 +15,6 @@ from typing import Sequence
 from typing import Tuple
 from typing import TypeVar
 from typing import Union
-from urllib.parse import urljoin
 
 try:
     import numpy as np
@@ -611,8 +609,11 @@ def dtype_to_sql(dtype: str, default: Any = None) -> str:
 
 def signature_to_sql(
     signature: Dict[str, Any],
-    base_url: Optional[str] = None,
+    url: Optional[str] = None,
     data_format: str = 'rowdat_1',
+    app_mode: str = 'remote',
+    link: Optional[str] = None,
+    replace: bool = False,
 ) -> str:
     '''
     Convert a dictionary function signature into SQL.
@@ -646,37 +647,27 @@ def signature_to_sql(
     host = os.environ.get('SINGLESTOREDB_EXT_HOST', '127.0.0.1')
     port = os.environ.get('SINGLESTOREDB_EXT_PORT', '8000')
 
-    url = urljoin(base_url or f'https://{host}:{port}', signature['endpoint'])
+    if app_mode.lower() == 'remote':
+        url = url or f'https://{host}:{port}/invoke'
+    elif url is None:
+        raise ValueError('url can not be `None`')
 
     database = ''
     if signature.get('database'):
         database = escape_name(signature['database']) + '.'
 
-    replace = 'OR REPLACE ' if signature.get('replace') else ''
+    or_replace = 'OR REPLACE ' if (bool(signature.get('replace')) or replace) else ''
 
-    return (
-        f'CREATE {replace}EXTERNAL FUNCTION {database}{escape_name(signature["name"])}' +
-        '(' + ', '.join(args) + ')' + returns +
-        f' AS REMOTE SERVICE "{url}" FORMAT {data_format.upper()};'
-    )
-
-
-def func_to_env(func: Callable[..., Any]) -> str:
-    # TODO: multiple functions
-    signature = get_signature(func)
-    env_name = signature['environment']
-    replace = 'OR REPLACE ' if signature.get('replace') else ''
-    packages = ', '.join(escape_item(x, 'utf8') for x in signature.get('packages', []))
-    resources = ', '.join(escape_item(x, 'utf8') for x in signature.get('resources', []))
-    code = inspect.getsource(func)
+    link_str = ''
+    if link:
+        if not re.match(r'^[\w_]+$', link):
+            raise ValueError(f'invalid LINK name: {link}')
+        link_str = f' LINK {link}'
 
     return (
-        f'CREATE {replace}ENVIRONMENT {env_name} LANGUAGE PYTHON ' +
-        'USING EXPORTS ' + escape_name(func.__name__) + ' ' +
-        (f'\n    PACKAGES ({packages}) ' if packages else '') +
-        (f'\n    RESOURCES ({resources}) ' if resources else '') +
-        '\n    AS CLOUD SERVICE' +
-        '\n    BEGIN\n' +
-        textwrap.indent(code, '    ') +
-        '    END;'
+        f'CREATE {or_replace}EXTERNAL FUNCTION ' +
+        f'{database}{escape_name(signature["name"])}' +
+        '(' + ', '.join(args) + ')' + returns +
+        f' AS {app_mode.upper()} SERVICE "{url}" FORMAT {data_format.upper()}'
+        f'{link_str};'
     )

singlestoredb/management/__init__.py

@@ -2,4 +2,5 @@
 from .cluster import manage_cluster
 from .manager import get_organization
 from .manager import get_token
+from .workspace import get_secret
 from .workspace import manage_workspaces

singlestoredb/management/manager.py

@@ -44,10 +44,10 @@ class Manager(object):
     """SingleStoreDB manager base class."""
 
     #: Management API version if none is specified.
-    default_version = 'v1'
+    default_version = config.get_option('management.version')
 
     #: Base URL if none is specified.
-    default_base_url = 'https://api.singlestore.com'
+    default_base_url = config.get_option('management.base_url')
 
     #: Object type
     obj_type = ''
@@ -57,14 +57,15 @@ class Manager(object):
         base_url: Optional[str] = None, *, organization_id: Optional[str] = None,
     ):
         from .. import __version__ as client_version
-        access_token = (
+        new_access_token = (
             access_token or get_token()
         )
-        if not access_token:
+        if not new_access_token:
             raise ManagementError(msg='No management token was configured.')
+        self._is_jwt = not access_token and new_access_token and is_jwt(new_access_token)
         self._sess = requests.Session()
         self._sess.headers.update({
-            'Authorization': f'Bearer {access_token}',
+            'Authorization': f'Bearer {new_access_token}',
             'Content-Type': 'application/json',
             'Accept': 'application/json',
             'User-Agent': f'SingleStoreDB-Python/{client_version}',
@@ -116,6 +117,9 @@ class Manager(object):
         **kwargs: Any,
     ) -> requests.Response:
         """Perform HTTP request."""
+        # Refresh the JWT as needed
+        if self._is_jwt:
+            self._sess.headers.update({'Authorization': f'Bearer {get_token()}'})
         return getattr(self._sess, method.lower())(
             urljoin(self._base_url, path), *args, **kwargs,
         )

singlestoredb/management/workspace.py

@@ -7,6 +7,7 @@ import glob
 import io
 import os
 import re
+import socket
 import time
 from typing import Any
 from typing import BinaryIO
@@ -31,6 +32,11 @@ from .utils import ttl_property
 from .utils import vars_to_str
 
 
+def get_secret(name: str) -> str:
+    """Get a secret from the organization."""
+    return manage_workspaces().organization.get_secret(name).value
+
+
 class StageObject(object):
     """
     Stage file / folder object.
@@ -1334,7 +1340,7 @@ class WorkspaceGroup(object):
     def create_workspace(
         self, name: str, size: Optional[str] = None,
         wait_on_active: bool = False, wait_interval: int = 10,
-        wait_timeout: int = 600,
+        wait_timeout: int = 600, add_endpoint_to_firewall_ranges: bool = True,
     ) -> Workspace:
         """
         Create a new workspace.
@@ -1352,6 +1358,9 @@ class WorkspaceGroup(object):
             if wait=True
         wait_interval : int, optional
             Number of seconds between each polling interval
+        add_endpoint_to_firewall_ranges : bool, optional
+            Should the workspace endpoint be added to the workspace group
+            firewall ranges?
 
         Returns
         -------
@@ -1362,11 +1371,18 @@ class WorkspaceGroup(object):
             raise ManagementError(
                 msg='No workspace manager is associated with this object.',
             )
-        return self._manager.create_workspace(
+
+        out = self._manager.create_workspace(
             name=name, workspace_group=self, size=size, wait_on_active=wait_on_active,
             wait_interval=wait_interval, wait_timeout=wait_timeout,
         )
 
+        if add_endpoint_to_firewall_ranges and out.endpoint is not None:
+            ip_address = '{}/32'.format(socket.gethostbyname(out.endpoint))
+            self.update(firewall_ranges=self.firewall_ranges+[ip_address])
+
+        return out
+
     @property
     def workspaces(self) -> NamedList[Workspace]:
         """Return a list of available workspaces."""

singlestoredb/tests/test_ext_func.py

@@ -65,11 +65,13 @@ def start_http_server(database, data_format='rowdat_1'):
         time.sleep(3)
         retries -= 1
 
-    app = create_app(ext_funcs)
+    app = create_app(
+        ext_funcs,
+        url=f'http://{HTTP_HOST}:{port}/invoke',
+        data_format=data_format,
+    )
     app.register_functions(
-        base_url=f'http://{HTTP_HOST}:{port}',
         database=database,
-        data_format=data_format,
     )
 
     with s2.connect(database=database) as conn:

singlestoredb/tests/test_udf.py

@@ -412,17 +412,6 @@ class TestUDF(unittest.TestCase):
         assert to_sql(foo) == '`hello``_``world`(`x` BIGINT NOT NULL) ' \
                               'RETURNS BIGINT NOT NULL'
 
-        # Add database name
-        @udf(database='mydb')
-        def foo(x: int) -> int: ...
-        assert to_sql(foo) == '`mydb`.`foo`(`x` BIGINT NOT NULL) ' \
-                              'RETURNS BIGINT NOT NULL'
-
-        @udf(database='my`db')
-        def foo(x: int) -> int: ...
-        assert to_sql(foo) == '`my``db`.`foo`(`x` BIGINT NOT NULL) ' \
-                              'RETURNS BIGINT NOT NULL'
-
     def test_dtypes(self):
         assert dt.BOOL() == 'BOOL NULL'
         assert dt.BOOL(nullable=False) == 'BOOL NOT NULL'

{singlestoredb-1.0.1.dist-info → singlestoredb-1.0.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: singlestoredb
-Version: 1.0.1
+Version: 1.0.3
 Summary: Interface to the SingleStoreDB database and workspace management APIs
 Home-page: https://github.com/singlestore-labs/singlestoredb-python
 Author: SingleStore

{singlestoredb-1.0.1.dist-info → singlestoredb-1.0.3.dist-info}/RECORD

@@ -1,7 +1,7 @@
-_singlestoredb_accel.pyd,sha256=Bw78h2Wr9jGrGf6WvCak-kzFzCwVrqeoFl3GneP6cIw,56832
-singlestoredb/__init__.py,sha256=gnTnujHQYcpgy3WHCVWTdBGLRZuNfeCBSaooiorLWY0,1697
+_singlestoredb_accel.pyd,sha256=DGQ8PSMhu2SXv5mTsAzJOEazCtHjhpuy56HqiyvNbnk,56832
+singlestoredb/__init__.py,sha256=gVaB4bDI0GkKtW3L3GT1bRcQptHqNSPi2Ml83c2XmVs,1697
 singlestoredb/auth.py,sha256=RmYiH0Wlc2RXc4pTlRMysxtBI445ggCIwojWKC_eDLE,7844
-singlestoredb/config.py,sha256=Uq7LWKlIwMAJC3_G3j5dUIQVJf2JzbDVtRK6votjuDw,7746
+singlestoredb/config.py,sha256=DruELa5JIPiJkMDF4WxJdB-QvnzdjTeNoja_Jp7kR04,8138
 singlestoredb/connection.py,sha256=gSKmMN1d_ZSXVTmzfuHK3BXcLyNGiaSi2Vy6z7nymjg,45657
 singlestoredb/converters.py,sha256=stCOsEcZmkfG_Y3jtfIzUmDTt_aMKv4Ir1EDR_r0ceg,13075
 singlestoredb/exceptions.py,sha256=WCCJrNSsU-hD-621Jpd6bwmvGftQ7byXkk-XKXlaxpg,3354
@@ -9,13 +9,14 @@ singlestoredb/pytest.py,sha256=TH364xRCN7_QaN0oRQDHixrEcDx_ZBgu3bmY0tvKrYU,9357
 singlestoredb/types.py,sha256=Lv0BEQl6aSZBiAe0OSI07FEJhcHZ9HX45iT9NU_mxHQ,10334
 singlestoredb/alchemy/__init__.py,sha256=bUmCl1xUn2v36RMbXLIrvgKzZSqx71mp1ReUw9JeVA8,2613
 singlestoredb/functions/__init__.py,sha256=EVxqWOCcXiIX4Yj7rljAYBBoVbTvm2KSuKSkMBDnEeU,42
-singlestoredb/functions/decorator.py,sha256=b0aQ68lllxNokC-bvD27IkFKA0OD-7PzxHfLnxCSJw4,7719
+singlestoredb/functions/decorator.py,sha256=M103c1JAZfyGFQAU4uJ_J8XGGH3InhcfrNUCoEORNFQ,5335
 singlestoredb/functions/dtypes.py,sha256=gwDokEe7P8gvvld158CWoCKsb6Sv-77FzeKXMQiOHEw,38351
-singlestoredb/functions/signature.py,sha256=5RGRspj_au0YJfRFeUjRmQMFzvg3a0RCpMb_uA1Poi8,20025
+singlestoredb/functions/signature.py,sha256=glxf8wVhwpsLOu9s9UEXPaXzBWvl_XN683_dpFyiQ6s,19539
 singlestoredb/functions/ext/__init__.py,sha256=NrwbyL86NeG_Kv1N23R4VwL1Ap-pY9Z1By6vnKzyZBE,68
 singlestoredb/functions/ext/arrow.py,sha256=mQhwaMpvCH_dP92WIhP_j-stu272n4UAHsFUOBTgnq0,9436
-singlestoredb/functions/ext/asgi.py,sha256=awkfdtlMnJzomcssmacUFtrhCIzMZH5WkC7pxMxNF8M,19494
+singlestoredb/functions/ext/asgi.py,sha256=fPNT4KrCXPEorA6WMBFFOpj87HPSAP70Xwd7z6fMLhI,22599
 singlestoredb/functions/ext/json.py,sha256=h0n4BZCbOWUM2le6wiysZR16bku_xgOMGmjN4Qx3Hw4,10799
+singlestoredb/functions/ext/mmap.py,sha256=0FbDY1IHF-27ZepnMuvvL9SP7tqjvPv7_WMmLQRo_3Q,10065
 singlestoredb/functions/ext/rowdat_1.py,sha256=24mNX-1Z-ala6QwSj4_WPNk4oxbruRtCBXZoFIYqUt8,23018
 singlestoredb/fusion/__init__.py,sha256=FHWtrg6OJFTf6Ye197V5sU6ssryr2h6FBcDIgXP7-H4,367
 singlestoredb/fusion/graphql.py,sha256=SHqsPe4xgawdsTPHEtJGQlybYGWqPrGMmyK-v20RLac,5420
@@ -28,14 +29,14 @@ singlestoredb/fusion/handlers/utils.py,sha256=7xWb_1mJzxW0po9iHVY2ZVnRvHIQgOlKZQ
 singlestoredb/fusion/handlers/workspace.py,sha256=ulxyFFLVpam83fPHI87Bwqc2V6AoGGHM-W8en3xq75s,11754
 singlestoredb/http/__init__.py,sha256=4cEDvLloGc3LSpU-PnIwacyu0n5oIIIE6xk2SPyWD_w,939
 singlestoredb/http/connection.py,sha256=8JO08meeJFHtTEqFSb_ju3dCVPLL8jQ4CLESeJ-JRyw,38602
-singlestoredb/management/__init__.py,sha256=kdKM-wErALWaEHYBLx_raHkJ8xUvUPflhOk8OBJyn58,173
+singlestoredb/management/__init__.py,sha256=1xAck9ehp2aGsDMAk5paS1Ek1EdjkDlpG1GqMJwm7h0,208
 singlestoredb/management/billing_usage.py,sha256=0UHFSPCrN0nyeGFFM-HXS3NP8pYmYo2BCCahDEPXvzg,3883
 singlestoredb/management/cluster.py,sha256=0GhpuSt_rcFz5f1hzcRHK911KWFewljlV4GFtckB8uM,14822
-singlestoredb/management/manager.py,sha256=w9QVuj-zn5ubLOLx36bsW-Ok6KFzZRMU8ElG6hEJY9k,8812
+singlestoredb/management/manager.py,sha256=QpWgu9W9n_HqxDJ4lAAFN7n1fhLB_BYkPy0_9uhGJvY,9107
 singlestoredb/management/organization.py,sha256=EywczC4uU1i70x_OkSqKnP6V_9D-ZuHBlETCogvJk_8,5104
 singlestoredb/management/region.py,sha256=oGoLLS88dE1GmY7GCc0BV7X3f7bWwKQyeXOVBFmK9Pk,1678
 singlestoredb/management/utils.py,sha256=Q9GXxbbSbYYFLHeCI9LSR1FqZarSAMvVdrMxq3JfjeQ,8613
-singlestoredb/management/workspace.py,sha256=Dnxk19hdKwd8SPO9Bf50fVnr-DQMW4AwvATpQEcTqkw,52966
+singlestoredb/management/workspace.py,sha256=1ymFz45UH5VtkTgVO9bEtzf9UeWxQawUSQILumxMXXc,53596
 singlestoredb/mysql/__init__.py,sha256=CbpwzNUJPAmKPpIobC0-ugBta_RgHCMq7X7N75QLReY,4669
 singlestoredb/mysql/_auth.py,sha256=YaqqyvAHmeraBv3BM207rNveUVPM-mPnW20ts_ynVWg,8341
 singlestoredb/mysql/charset.py,sha256=mnCdMpvdub1S2mm2PSk2j5JddgsWRjsVLtGx-y9TskE,10724
@@ -85,7 +86,7 @@ singlestoredb/tests/test_config.py,sha256=Ad0PDmCnJMOyy9f7WTKiRasSR_3mYRByUlSb7k
 singlestoredb/tests/test_connection.py,sha256=0GRvsvUz8G2I5ah0lHI97XUVv6UI13A1D5UNHk7RRmc,52215
 singlestoredb/tests/test_dbapi.py,sha256=cNJoTEZvYG7ckcwT7xqlkJX-2TDEYGTDDU1Igucp48k,679
 singlestoredb/tests/test_exceptions.py,sha256=vscMYmdOJr0JmkTAJrNI2w0Q96Nfugjkrt5_lYnw8i0,1176
-singlestoredb/tests/test_ext_func.py,sha256=ZqC9-N0yBUpt670Iaf-SVvu54I_cXu9l9XF-vQSdgGk,38521
+singlestoredb/tests/test_ext_func.py,sha256=c_K1B62l1kLDieuoz9GbYcGBgmLw3eja7JqwGD-AqBE,38540
 singlestoredb/tests/test_ext_func_data.py,sha256=1iqc9urXqnb1BM5gIQxzK_Q1dnsw3aDflIFMFQfSX28,48794
 singlestoredb/tests/test_fusion.py,sha256=oFfn7vtdMeTEl02JC74JcQHSA6V1BgzAkioBSnruwPs,15565
 singlestoredb/tests/test_http.py,sha256=7hwXe61hlUes3nji0MTTZweo94tJAlJ-vA5ct9geXFQ,8868
@@ -93,7 +94,7 @@ singlestoredb/tests/test_management.py,sha256=WmgsCCpPQTks8WyeD6ZO5ID0B_3GKZphW8
 singlestoredb/tests/test_plugin.py,sha256=P1nXLnTafaHkHN-6bVbGryxTu7OWJPU9SYFZ_WQUwq8,845
 singlestoredb/tests/test_results.py,sha256=Zg1ynZFRZqalAMfNLOU5C6BDXaox6JxrKm_XZwVNFcg,6753
 singlestoredb/tests/test_types.py,sha256=YeVE6KPqlqzJke-4hbRmc8ko1E7RLHu5S8qLg04Bl5Y,4632
-singlestoredb/tests/test_udf.py,sha256=_YUKCAfSii5I4tIrqbaVrBzBoVUiELrgvLVO_vkQSPM,29188
+singlestoredb/tests/test_udf.py,sha256=pac1Qp1JPcUgXB1-xBBMRCqWD17IBWMQ8TFe6LE3iLo,28762
 singlestoredb/tests/test_xdict.py,sha256=5ArRJqd5aNXkPK7Y6sFeRbqZ59MZ1YaGBpSlDAbBrjM,10741
 singlestoredb/tests/utils.py,sha256=1ZliGv1gqkKEzb9OhRSxTaWg7md_Z4htIxTMiOjyHj0,4807
 singlestoredb/tests/ext_funcs/__init__.py,sha256=mKa-vOh5D8M03kgKTQyvOB74X-Of0nl-mcmJBFzRW0c,9675
@@ -104,9 +105,9 @@ singlestoredb/utils/debug.py,sha256=y7dnJeJGt3U_BWXz9pLt1qNQREpPtumYX_sk1DiqG6Y,
 singlestoredb/utils/mogrify.py,sha256=gCcn99-vgsGVjTUV7RHJ6hH4vCNrsGB_Xo4z8kiSPDQ,4201
 singlestoredb/utils/results.py,sha256=ely2XVAHHejObjLibS3UcrPOuCO2g5aRtA3PxAMtE-g,5432
 singlestoredb/utils/xdict.py,sha256=-wi1lSPTnY99fhVMBhPKJ8cCsQhNG4GMUfkEBDKYgCw,13321
-singlestoredb-1.0.1.dist-info/LICENSE,sha256=Bojenzui8aPNjlF3w4ojguDP7sTf8vFV_9Gc2UAG1sg,11542
-singlestoredb-1.0.1.dist-info/METADATA,sha256=9c4y611NWsrMV-dU8ZHKCzxNU0vQjdb0HwvuE-Zgllg,5654
-singlestoredb-1.0.1.dist-info/WHEEL,sha256=UyMHzmWA0xVqVPKfTiLs2eN3OWWZUl-kQemNbpIqlKo,100
-singlestoredb-1.0.1.dist-info/entry_points.txt,sha256=bSLaTWB5zGjpVYPAaI46MkkDup0su-eb3uAhCNYuRV0,48
-singlestoredb-1.0.1.dist-info/top_level.txt,sha256=SDtemIXf-Kp-_F2f_S6x0db33cHGOILdAEsIQZe2LZc,35
-singlestoredb-1.0.1.dist-info/RECORD,,
+singlestoredb-1.0.3.dist-info/LICENSE,sha256=Bojenzui8aPNjlF3w4ojguDP7sTf8vFV_9Gc2UAG1sg,11542
+singlestoredb-1.0.3.dist-info/METADATA,sha256=2gEE1suDB6kWGjr0Rhcboe-ELCcD4mrkrkI-zeojvfE,5654
+singlestoredb-1.0.3.dist-info/WHEEL,sha256=UyMHzmWA0xVqVPKfTiLs2eN3OWWZUl-kQemNbpIqlKo,100
+singlestoredb-1.0.3.dist-info/entry_points.txt,sha256=bSLaTWB5zGjpVYPAaI46MkkDup0su-eb3uAhCNYuRV0,48
+singlestoredb-1.0.3.dist-info/top_level.txt,sha256=SDtemIXf-Kp-_F2f_S6x0db33cHGOILdAEsIQZe2LZc,35
+singlestoredb-1.0.3.dist-info/RECORD,,