singlestoredb 1.4.0__tar.gz → 1.4.2__tar.gz

{singlestoredb-1.4.0/singlestoredb.egg-info → singlestoredb-1.4.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: singlestoredb
-Version: 1.4.0
+Version: 1.4.2
 Summary: Interface to the SingleStoreDB database and workspace management APIs
 Home-page: https://github.com/singlestore-labs/singlestoredb-python
 Author: SingleStore

{singlestoredb-1.4.0 → singlestoredb-1.4.2}/accel.c

@@ -421,6 +421,7 @@ typedef struct {
     PyObject *float64;
     PyObject *unpack;
     PyObject *decode;
+    PyObject *frombuffer;
 } PyStrings;
 
 static PyStrings PyStr = {0};
@@ -437,6 +438,7 @@ typedef struct {
     PyObject *datetime_datetime;
     PyObject *collections_namedtuple;
     PyObject *numpy_array;
+    PyObject *numpy_frombuffer;
     PyObject *numpy_vectorize;
     PyObject *pandas_DataFrame;
     PyObject *polars_DataFrame;
@@ -521,6 +523,9 @@ int ensure_numpy() {
     PyFunc.numpy_array = PyObject_GetAttr(numpy_mod, PyStr.array);
     if (!PyFunc.numpy_array) goto error;
 
+    PyFunc.numpy_frombuffer = PyObject_GetAttr(numpy_mod, PyStr.frombuffer);
+    if (!PyFunc.numpy_frombuffer) goto error;
+
     PyFunc.numpy_vectorize = PyObject_GetAttr(numpy_mod, PyStr.vectorize);
     if (!PyFunc.numpy_vectorize) goto error;
 
@@ -1476,7 +1481,7 @@ static PyObject *read_row_from_packet(
     PyObject *py_str = NULL;
     PyObject *py_memview = NULL;
     char end = '\0';
-    char *cast_type_codes[] = {"", "f", "d", "b", "h", "l", "q"};
+    char *cast_type_codes[] = {"", "f", "d", "b", "h", "i", "q"};
     int item_type_lengths[] = {0, 4, 8, 1, 2, 4, 8};
 
     int sign = 1;
@@ -1773,17 +1778,20 @@ static PyObject *read_row_from_packet(
                 case MYSQL_TYPE_INT16_VECTOR:
                 case MYSQL_TYPE_INT32_VECTOR:
                 case MYSQL_TYPE_INT64_VECTOR:
+                {
+                    int type_idx = py_state->type_codes[i] % 1000;
+
                     if (ensure_numpy() == 0) {
-                        py_memview = PyMemoryView_FromMemory(out, out_l, PyBUF_WRITE);
+                        py_memview = PyBytes_FromStringAndSize(out, out_l);
                         if (!py_memview) goto error;
 
-                        py_item = create_numpy_array(
-                            py_memview,
-                            cast_type_codes[py_state->type_codes[i] % 1000],
-                            py_state->type_codes[i],
-                            NULL
+                        CHECKRC(PyTuple_SetItem(PyObj.create_numpy_array_args, 0, py_memview));
+
+                        py_item = PyObject_Call(
+                            PyFunc.numpy_frombuffer,
+                            PyObj.create_numpy_array_args,
+                            PyObj.create_numpy_array_kwargs_vector[type_idx]
                         );
-                        Py_CLEAR(py_memview);
                         if (!py_item) goto error;
 
                     } else {
@@ -1791,7 +1799,7 @@ static PyObject *read_row_from_packet(
                         if (!py_memview) goto error;
 
                         CHECKRC(PyTuple_SetItem(PyObj.struct_unpack_args, 0,
-                                                PyUnicode_FromFormat("<%l%s", out_l / item_type_lengths[i], cast_type_codes[i])));
+                                                PyUnicode_FromFormat("<%ld%s", out_l / item_type_lengths[type_idx], cast_type_codes[type_idx])));
                         CHECKRC(PyTuple_SetItem(PyObj.struct_unpack_args, 1, py_memview));
 
                         py_item = PyObject_Call(
@@ -1803,6 +1811,7 @@ static PyObject *read_row_from_packet(
                     }
 
                     break;
+                }
 
                 case MYSQL_TYPE_BSON:
                     py_item = PyBytes_FromStringAndSize(out, out_l);
@@ -4614,6 +4623,7 @@ PyMODINIT_FUNC PyInit__singlestoredb_accel(void) {
     PyStr.float64 = PyUnicode_FromString("float64");
     PyStr.unpack = PyUnicode_FromString("unpack");
     PyStr.decode = PyUnicode_FromString("decode");
+    PyStr.frombuffer = PyUnicode_FromString("frombuffer");
 
     PyObject *decimal_mod = PyImport_ImportModule("decimal");
     if (!decimal_mod) goto error;

{singlestoredb-1.4.0 → singlestoredb-1.4.2}/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = singlestoredb
-version = 1.4.0
+version = 1.4.2
 description = Interface to the SingleStoreDB database and workspace management APIs
 long_description = file: README.md
 long_description_content_type = text/markdown

{singlestoredb-1.4.0 → singlestoredb-1.4.2}/singlestoredb/__init__.py

@@ -13,7 +13,7 @@ Examples
 
 """
 
-__version__ = '1.4.0'
+__version__ = '1.4.2'
 
 from typing import Any
 

{singlestoredb-1.4.0 → singlestoredb-1.4.2}/singlestoredb/connection.py

@@ -1359,6 +1359,10 @@ def connect(
     results_type : str, optional
         The form of the query results: tuples, namedtuples, dicts,
         numpy, polars, pandas, arrow
+    buffered : bool, optional
+        Should the entire query result be buffered in memory? This is the default
+        behavior which allows full cursor control of the result, but does consume
+        more memory.
     results_format : str, optional
         Deprecated. This option has been renamed to results_type.
     program_name : str, optional

{singlestoredb-1.4.0 → singlestoredb-1.4.2}/singlestoredb/functions/signature.py

@@ -45,15 +45,17 @@ if has_numpy:
         np.uint8: 'uint8',
         np.longlong: 'uint64',
         np.ulonglong: 'uint64',
-        np.unicode_: 'str',
         np.str_: 'str',
         np.bytes_: 'bytes',
-        np.float_: 'float64',
         np.float64: 'float64',
         np.float32: 'float32',
         np.float16: 'float16',
         np.double: 'float64',
     }
+    if hasattr(np, 'unicode_'):
+        numpy_type_map[np.unicode_] = 'str'
+    if hasattr(np, 'float_'):
+        numpy_type_map[np.float_] = 'float64'
 else:
     array_types = (Sequence,)
     numpy_type_map = {}

{singlestoredb-1.4.0 → singlestoredb-1.4.2}/singlestoredb/fusion/handler.py

@@ -28,8 +28,23 @@ CORE_GRAMMAR = r'''
     eq = ws* "=" ws*
     open_paren = ws* "(" ws*
     close_paren = ws* ")" ws*
+    open_repeats = ws* ~r"[\(\[\{]" ws*
+    close_repeats = ws* ~r"[\)\]\}]" ws*
     select = ~r"SELECT"i ws+ ~r".+" ws*
-'''
+
+    json = ws* json_object ws*
+    json_object = ~r"{\s*" json_members? ~r"\s*}"
+    json_members = json_mapping (~r"\s*,\s*" json_mapping)*
+    json_mapping = json_string ~r"\s*:\s*" json_value
+    json_array = ~r"\[\s*" json_items? ~r"\s*\]"
+    json_items = json_value (~r"\s*,\s*" json_value)*
+    json_value = json_object / json_array / json_string / json_true_val / json_false_val / json_null_val / json_number
+    json_true_val = "true"
+    json_false_val = "false"
+    json_null_val = "null"
+    json_string = ~r"\"[ !#-\[\]-\U0010ffff]*(?:\\(?:[\"\\/bfnrt]|u[0-9A-Fa-f]{4})[ !#-\[\]-\U0010ffff]*)*\""
+    json_number = ~r"-?(0|[1-9][0-9]*)(\.\d*)?([eE][-+]?\d+)?"
+'''  # noqa: E501
 
 BUILTINS = {
     '<order-by>': r'''
@@ -47,6 +62,7 @@ BUILTINS = {
     ''',
     '<integer>': '',
     '<number>': '',
+    '<json>': '',
 }
 
 BUILTIN_DEFAULTS = {  # type: ignore
@@ -54,9 +70,36 @@ BUILTIN_DEFAULTS = {  # type: ignore
     'like': None,
     'extended': False,
     'limit': None,
+    'json': {},
+}
+
+_json_unesc_re = re.compile(r'\\(["/\\bfnrt]|u[0-9A-Fa-f])')
+_json_unesc_map = {
+    '"': '"',
+    '/': '/',
+    '\\': '\\',
+    'b': '\b',
+    'f': '\f',
+    'n': '\n',
+    'r': '\r',
+    't': '\t',
 }
 
 
+def _json_unescape(m: Any) -> str:
+    c = m.group(1)
+    if c[0] == 'u':
+        return chr(int(c[1:], 16))
+    c2 = _json_unesc_map.get(c)
+    if not c2:
+        raise ValueError(f'invalid escape sequence: {m.group(0)}')
+    return c2
+
+
+def json_unescape(s: str) -> str:
+    return _json_unesc_re.sub(_json_unescape, s[1:-1])
+
+
 def get_keywords(grammar: str) -> Tuple[str, ...]:
     """Return all all-caps words from the beginning of the line."""
     m = re.match(r'^\s*((?:[A-Z0-9_]+)(\s+|$|;))+', grammar)
@@ -89,7 +132,7 @@ def process_alternates(m: Any) -> str:
 def process_repeats(m: Any) -> str:
     """Add repeated patterns."""
     sql = m.group(1).strip()
-    return f'open_paren? {sql} ws* ( comma {sql} ws* )* close_paren?'
+    return f'open_repeats? {sql} ws* ( comma {sql} ws* )* close_repeats?'
 
 
 def lower_and_regex(m: Any) -> str:
@@ -639,6 +682,14 @@ class SQLHandler(NodeVisitor):
         """Close parenthesis."""
         return
 
+    def visit_open_repeats(self, node: Node, visited_children: Iterable[Any]) -> Any:
+        """Open repeat grouping."""
+        return
+
+    def visit_close_repeats(self, node: Node, visited_children: Iterable[Any]) -> Any:
+        """Close repeat grouping."""
+        return
+
     def visit_init(self, node: Node, visited_children: Iterable[Any]) -> Any:
         """Entry point of the grammar."""
         _, out, *_ = visited_children
@@ -662,6 +713,57 @@ class SQLHandler(NodeVisitor):
             ascending.append(value[1].upper().startswith('A'))
         return {'order_by': {'by': by, 'ascending': ascending}}
 
+    def _delimited(self, node: Node, children: Iterable[Any]) -> Any:
+        children = list(children)
+        items = [children[0]]
+        items.extend(item for _, item in children[1])
+        return items
+
+    def _atomic(self, node: Node, children: Iterable[Any]) -> Any:
+        return list(children)[0]
+
+    # visitors
+    visit_json_value = _atomic
+    visit_json_members = visit_json_items = _delimited
+
+    def visit_json_object(self, node: Node, children: Iterable[Any]) -> Any:
+        _, members, _ = children
+        if isinstance(members, list):
+            members = members[0]
+        else:
+            members = []
+        members = [x for x in members if x != '']
+        return dict(members)
+
+    def visit_json_array(self, node: Node, children: Iterable[Any]) -> Any:
+        _, values, _ = children
+        if isinstance(values, list):
+            values = values[0]
+        else:
+            values = []
+        return values
+
+    def visit_json_mapping(self, node: Node, children: Iterable[Any]) -> Any:
+        key, _, value = children
+        return key, value
+
+    def visit_json_string(self, node: Node, children: Iterable[Any]) -> Any:
+        return json_unescape(node.text)
+
+    def visit_json_number(self, node: Node, children: Iterable[Any]) -> Any:
+        if '.' in node.text:
+            return float(node.text)
+        return int(node.text)
+
+    def visit_json_true_val(self, node: Node, children: Iterable[Any]) -> Any:
+        return True
+
+    def visit_json_false_val(self, node: Node, children: Iterable[Any]) -> Any:
+        return False
+
+    def visit_json_null_val(self, node: Node, children: Iterable[Any]) -> Any:
+        return None
+
     def generic_visit(self, node: Node, visited_children: Iterable[Any]) -> Any:
         """
         Handle all undefined rules.
@@ -675,6 +777,9 @@ class SQLHandler(NodeVisitor):
         rule can have repeated values, a list of values is returned.
 
         """
+        if node.expr_name.startswith('json'):
+            return visited_children or node.text
+
         # Call a grammar rule
         if node.expr_name in type(self).rule_info:
             n_keywords = type(self).rule_info[node.expr_name]['n_keywords']

{singlestoredb-1.4.0 → singlestoredb-1.4.2}/singlestoredb/management/__init__.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 from .cluster import manage_cluster
-from .manager import get_organization
 from .manager import get_token
+from .workspace import get_organization
 from .workspace import get_secret
 from .workspace import get_stage
 from .workspace import manage_workspaces

{singlestoredb-1.4.0 → singlestoredb-1.4.2}/singlestoredb/management/cluster.py

@@ -8,6 +8,7 @@ from typing import List
 from typing import Optional
 from typing import Union
 
+from .. import config
 from .. import connection
 from ..exceptions import ManagementError
 from .manager import Manager
@@ -332,7 +333,7 @@ class ClusterManager(Manager):
     default_version = 'v0beta'
 
     #: Base URL if none is specified.
-    default_base_url = 'https://api.singlestore.com'
+    default_base_url = config.get_option('management.base_url')
 
     #: Object type
     obj_type = 'cluster'
@@ -425,8 +426,8 @@ class ClusterManager(Manager):
 
 def manage_cluster(
     access_token: Optional[str] = None,
-    version: str = ClusterManager.default_version,
-    base_url: str = ClusterManager.default_base_url,
+    version: Optional[str] = None,
+    base_url: Optional[str] = None,
     *,
     organization_id: Optional[str] = None,
 ) -> ClusterManager:

{singlestoredb-1.4.0 → singlestoredb-1.4.2}/singlestoredb/management/manager.py

@@ -15,7 +15,6 @@ import requests
 
 from .. import config
 from ..exceptions import ManagementError
-from .utils import get_organization
 from .utils import get_token
 
 
@@ -24,7 +23,7 @@ def set_organization(kwargs: Dict[str, Any]) -> None:
     if kwargs.get('params', {}).get('organizationID', None):
         return
 
-    org = get_organization()
+    org = os.environ.get('SINGLESTOREDB_ORGANIZATION')
     if org:
         if 'params' not in kwargs:
             kwargs['params'] = {}

{singlestoredb-1.4.0 → singlestoredb-1.4.2}/singlestoredb/management/utils.py

@@ -2,6 +2,7 @@
 """SingleStoreDB Cluster Management."""
 import datetime
 import functools
+import itertools
 import os
 import re
 import sys
@@ -12,6 +13,7 @@ from typing import List
 from typing import Mapping
 from typing import Optional
 from typing import SupportsIndex
+from typing import Tuple
 from typing import TypeVar
 from typing import Union
 from urllib.parse import urlparse
@@ -20,6 +22,7 @@ import jwt
 
 from .. import converters
 from ..config import get_option
+from ..utils import events
 
 JSON = Union[str, List[str], Dict[str, 'JSON']]
 JSONObj = Dict[str, JSON]
@@ -117,6 +120,66 @@ class NamedList(List[T]):
             raise
 
 
+def _setup_authentication_info_handler() -> Callable[..., Dict[str, Any]]:
+    """Setup authentication info event handler."""
+
+    authentication_info: List[Tuple[str, Any]] = []
+
+    def handle_authentication_info(msg: Dict[str, Any]) -> None:
+        """Handle authentication info events."""
+        nonlocal authentication_info
+        if msg.get('name', '') != 'singlestore.portal.authentication_updated':
+            return
+        authentication_info = list(msg.get('data', {}).items())
+
+    events.subscribe(handle_authentication_info)
+
+    def handle_connection_info(msg: Dict[str, Any]) -> None:
+        """Handle connection info events."""
+        nonlocal authentication_info
+        if msg.get('name', '') != 'singlestore.portal.connection_updated':
+            return
+        data = msg.get('data', {})
+        out = {}
+        if 'user' in data:
+            out['user'] = data['user']
+        if 'password' in data:
+            out['password'] = data['password']
+        authentication_info = list(out.items())
+
+    events.subscribe(handle_authentication_info)
+
+    def get_env() -> List[Tuple[str, Any]]:
+        conn = {}
+        url = os.environ.get('SINGLESTOREDB_URL') or get_option('host')
+        if url:
+            urlp = urlparse(url, scheme='singlestoredb', allow_fragments=True)
+            conn = dict(
+                user=urlp.username or None,
+                password=urlp.password or None,
+            )
+
+        return [
+            x for x in dict(
+                **conn,
+            ).items() if x[1] is not None
+        ]
+
+    def get_authentication_info(include_env: bool = True) -> Dict[str, Any]:
+        """Return authentication info from event."""
+        return dict(
+            itertools.chain(
+                (get_env() if include_env else []),
+                authentication_info,
+            ),
+        )
+
+    return get_authentication_info
+
+
+get_authentication_info = _setup_authentication_info_handler()
+
+
 def get_token() -> Optional[str]:
     """Return the token for the Management API."""
     # See if an API key is configured
@@ -124,18 +187,11 @@ def get_token() -> Optional[str]:
     if tok:
         return tok
 
-    url = os.environ.get('SINGLESTOREDB_URL')
-    if not url:
-        # See if the connection URL contains a JWT
-        url = get_option('host')
-        if not url:
-            return None
-
-    urlp = urlparse(url, scheme='singlestoredb', allow_fragments=True)
-    if urlp.password:
+    tok = get_authentication_info(include_env=True).get('password')
+    if tok:
         try:
-            jwt.decode(urlp.password, options={'verify_signature': False})
-            return urlp.password
+            jwt.decode(tok, options={'verify_signature': False})
+            return tok
         except jwt.DecodeError:
             pass
 
@@ -143,15 +199,6 @@ def get_token() -> Optional[str]:
     return None
 
 
-def get_organization() -> Optional[str]:
-    """Return the organization for the current token or environment."""
-    org = os.environ.get('SINGLESTOREDB_ORGANIZATION')
-    if org:
-        return org
-
-    return None
-
-
 def enable_http_tracing() -> None:
     """Enable tracing of HTTP requests."""
     import logging
@@ -269,7 +316,7 @@ def snake_to_camel(s: Optional[str], cap_first: bool = False) -> Optional[str]:
     """Convert snake-case to camel-case."""
     if s is None:
         return None
-    out = re.sub(r'_[A-Za-z]', _upper_match, s.lower())
+    out = re.sub(r'_([A-Za-z])', _upper_match, s.lower())
     if cap_first and out:
         return out[0].upper() + out[1:]
     return out

{singlestoredb-1.4.0 → singlestoredb-1.4.2}/singlestoredb/management/workspace.py

@@ -17,6 +17,7 @@ from typing import Optional
 from typing import TextIO
 from typing import Union
 
+from .. import config
 from .. import connection
 from ..exceptions import ManagementError
 from .billing_usage import BillingUsageItem
@@ -1690,10 +1691,10 @@ class WorkspaceManager(Manager):
     """
 
     #: Workspace management API version if none is specified.
-    default_version = 'v1'
+    default_version = config.get_option('management.version')
 
     #: Base URL if none is specified.
-    default_base_url = 'https://api.singlestore.com'
+    default_base_url = config.get_option('management.base_url')
 
     #: Object type
     obj_type = 'workspace'
@@ -1906,8 +1907,8 @@ class WorkspaceManager(Manager):
 
 def manage_workspaces(
     access_token: Optional[str] = None,
-    version: str = WorkspaceManager.default_version,
-    base_url: str = WorkspaceManager.default_base_url,
+    version: Optional[str] = None,
+    base_url: Optional[str] = None,
     *,
     organization_id: Optional[str] = None,
 ) -> WorkspaceManager:

{singlestoredb-1.4.0 → singlestoredb-1.4.2}/singlestoredb/mysql/connection.py

@@ -13,6 +13,8 @@ import struct
 import sys
 import traceback
 import warnings
+from typing import Any
+from typing import Dict
 from typing import Iterable
 
 try:
@@ -21,6 +23,7 @@ except (ImportError, ModuleNotFoundError):
     _singlestoredb_accel = None
 
 from . import _auth
+from ..utils import events
 
 from .charset import charset_by_name, charset_by_id
 from .constants import CLIENT, COMMAND, CR, ER, FIELD_TYPE, SERVER_STATUS
@@ -100,6 +103,19 @@ TEXT_TYPES = {
     FIELD_TYPE.VAR_STRING,
     FIELD_TYPE.VARCHAR,
     FIELD_TYPE.GEOMETRY,
+    FIELD_TYPE.BSON,
+    FIELD_TYPE.FLOAT32_VECTOR_JSON,
+    FIELD_TYPE.FLOAT64_VECTOR_JSON,
+    FIELD_TYPE.INT8_VECTOR_JSON,
+    FIELD_TYPE.INT16_VECTOR_JSON,
+    FIELD_TYPE.INT32_VECTOR_JSON,
+    FIELD_TYPE.INT64_VECTOR_JSON,
+    FIELD_TYPE.FLOAT32_VECTOR,
+    FIELD_TYPE.FLOAT64_VECTOR,
+    FIELD_TYPE.INT8_VECTOR,
+    FIELD_TYPE.INT16_VECTOR,
+    FIELD_TYPE.INT32_VECTOR,
+    FIELD_TYPE.INT64_VECTOR,
 }
 
 UNSET = 'unset'
@@ -614,15 +630,22 @@ class Connection(BaseConnection):
                 if k not in self._connect_attrs:
                     self._connect_attrs[k] = v
 
+        self._is_committable = True
         self._in_sync = False
         self._track_env = bool(track_env) or self.host == 'singlestore.com'
         self._enable_extended_data_types = enable_extended_data_types
+        self._connection_info = {}
+        events.subscribe(self._handle_event)
 
         if defer_connect or self._track_env:
             self._sock = None
         else:
             self.connect()
 
+    def _handle_event(self, data: Dict[str, Any]) -> None:
+        if data.get('name', '') == 'singlestore.portal.connection_updated':
+            self._connection_info = dict(data)
+
     @property
     def messages(self):
         # TODO
@@ -766,7 +789,8 @@ class Connection(BaseConnection):
 
         """
         log_query('COMMIT')
-        if self.host == 'singlestore.com':
+        if not self._is_committable or self.host == 'singlestore.com':
+            self._is_committable = True
             return
         self._execute_command(COMMAND.COM_QUERY, 'COMMIT')
         self._read_ok_packet()
@@ -780,7 +804,8 @@ class Connection(BaseConnection):
 
         """
         log_query('ROLLBACK')
-        if self.host == 'singlestore.com':
+        if not self._is_committable or self.host == 'singlestore.com':
+            self._is_committable = True
             return
         self._execute_command(COMMAND.COM_QUERY, 'ROLLBACK')
         self._read_ok_packet()
@@ -858,9 +883,11 @@ class Connection(BaseConnection):
         #     print("DEBUG: sending query:", sql)
         handler = fusion.get_handler(sql)
         if handler is not None:
+            self._is_committable = False
             self._result = fusion.execute(self, sql, handler=handler)
             self._affected_rows = self._result.affected_rows
         else:
+            self._is_committable = True
             if isinstance(sql, str):
                 sql = sql.encode(self.encoding, 'surrogateescape')
             self._local_infile_stream = infile_stream
@@ -973,9 +1000,11 @@ class Connection(BaseConnection):
         if not self._track_env:
             return
 
-        url = os.environ.get('SINGLESTOREDB_URL')
+        url = self._connection_info.get('connection_url')
         if not url:
-            return
+            url = os.environ.get('SINGLESTOREDB_URL')
+            if not url:
+                return
 
         out = {}
         urlp = connection._parse_url(url)

{singlestoredb-1.4.0 → singlestoredb-1.4.2}/singlestoredb/notebook/__init__.py

@@ -7,6 +7,7 @@ from ._objects import secrets  # noqa: F401
 from ._objects import stage  # noqa: F401
 from ._objects import workspace  # noqa: F401
 from ._objects import workspace_group  # noqa: F401
+from ._portal import portal  # noqa: F401
 
 if 'SINGLESTOREDB_ORGANIZATION' not in _os.environ:
     _warnings.warn(