singlestoredb 1.7.2__py3-none-any.whl → 1.9.0__py3-none-any.whl

singlestoredb/__init__.py

@@ -13,7 +13,7 @@ Examples
 
 """
 
-__version__ = '1.7.2'
+__version__ = '1.9.0'
 
 from typing import Any
 
@@ -25,7 +25,7 @@ from .exceptions import (
     DataError, ManagementError,
 )
 from .management import (
-    manage_cluster, manage_workspaces,
+    manage_cluster, manage_workspaces, manage_files,
 )
 from .types import (
     Date, Time, Timestamp, DateFromTicks, TimeFromTicks, TimestampFromTicks,

singlestoredb/config.py

@@ -134,6 +134,12 @@ register_option(
     environ='SINGLESTOREDB_SSL_CIPHER',
 )
 
+register_option(
+    'tls_sni_servername', 'str', check_str, None,
+    'Sets TLS SNI servername',
+    environ='SINGLESTOREDB_TLS_SNI_SERVERNAME',
+)
+
 register_option(
     'ssl_disabled', 'bool', check_bool, False,
     'Disable SSL usage',

singlestoredb/connection.py

@@ -20,6 +20,7 @@ from typing import Sequence
 from typing import Tuple
 from typing import Union
 from urllib.parse import parse_qs
+from urllib.parse import unquote_plus
 from urllib.parse import urlparse
 
 import sqlparams
@@ -284,6 +285,15 @@ def _parse_url(url: str) -> Dict[str, Any]:
     if parts.scheme != 'singlestoredb':
         out['driver'] = parts.scheme.lower()
 
+    if out.get('user'):
+        out['user'] = unquote_plus(out['user'])
+
+    if out.get('password'):
+        out['password'] = unquote_plus(out['password'])
+
+    if out.get('database'):
+        out['database'] = unquote_plus(out['database'])
+
     # Convert query string to parameters
     out.update({k.lower(): v[-1] for k, v in parse_qs(parts.query).items()})
 
@@ -1288,6 +1298,7 @@ def connect(
     ssl_key: Optional[str] = None, ssl_cert: Optional[str] = None,
     ssl_ca: Optional[str] = None, ssl_disabled: Optional[bool] = None,
     ssl_cipher: Optional[str] = None, ssl_verify_cert: Optional[bool] = None,
+    tls_sni_servername: Optional[str] = None,
     ssl_verify_identity: Optional[bool] = None,
     conv: Optional[Dict[int, Callable[..., Any]]] = None,
     credential_type: Optional[str] = None,

singlestoredb/fusion/handler.py

@@ -11,6 +11,7 @@ from typing import Dict
 from typing import Iterable
 from typing import List
 from typing import Optional
+from typing import Set
 from typing import Tuple
 
 from parsimonious import Grammar
@@ -23,9 +24,9 @@ from ..connection import Connection
 
 CORE_GRAMMAR = r'''
     ws = ~r"(\s+|(\s*/\*.*\*/\s*)+)"
-    qs = ~r"\"([^\"]*)\"|'([^\']*)'|`([^\`]*)`|([A-Za-z0-9_\-\.]+)"
-    number = ~r"[-+]?(\d*\.)?\d+(e[-+]?\d+)?"i
-    integer = ~r"-?\d+"
+    qs = ~r"\"([^\"]*)\"|'([^\']*)'|([A-Za-z0-9_\-\.]+)|`([^\`]+)`" ws*
+    number = ~r"[-+]?(\d*\.)?\d+(e[-+]?\d+)?"i ws*
+    integer = ~r"-?\d+" ws*
     comma = ws* "," ws*
     eq = ws* "=" ws*
     open_paren = ws* "(" ws*
@@ -33,6 +34,10 @@ CORE_GRAMMAR = r'''
     open_repeats = ws* ~r"[\(\[\{]" ws*
     close_repeats = ws* ~r"[\)\]\}]" ws*
     select = ~r"SELECT"i ws+ ~r".+" ws*
+    table = ~r"(?:([A-Za-z0-9_\-]+)|`([^\`]+)`)(?:\.(?:([A-Za-z0-9_\-]+)|`([^\`]+)`))?" ws*
+    column = ~r"(?:([A-Za-z0-9_\-]+)|`([^\`]+)`)(?:\.(?:([A-Za-z0-9_\-]+)|`([^\`]+)`))?" ws*
+    link_name = ~r"(?:([A-Za-z0-9_\-]+)|`([^\`]+)`)(?:\.(?:([A-Za-z0-9_\-]+)|`([^\`]+)`))?" ws*
+    catalog_name = ~r"(?:([A-Za-z0-9_\-]+)|`([^\`]+)`)(?:\.(?:([A-Za-z0-9_\-]+)|`([^\`]+)`))?" ws*
 
     json = ws* json_object ws*
     json_object = ~r"{\s*" json_members? ~r"\s*}"
@@ -65,6 +70,13 @@ BUILTINS = {
     '<integer>': '',
     '<number>': '',
     '<json>': '',
+    '<table>': '',
+    '<column>': '',
+    '<catalog-name>': '',
+    '<link-name>': '',
+    '<file-type>': r'''
+    file_type = { FILE | FOLDER }
+    ''',
 }
 
 BUILTIN_DEFAULTS = {  # type: ignore
@@ -226,9 +238,13 @@ def build_syntax(grammar: str) -> str:
     # Split on ';' on a line by itself
     cmd, end = grammar.split(';', 1)
 
-    rules = {}
+    name = ''
+    rules: Dict[str, Any] = {}
     for line in end.split('\n'):
         line = line.strip()
+        if line.startswith('&'):
+            rules[name] += '\n' + line
+            continue
         if not line:
             continue
         name, value = line.split('=', 1)
@@ -239,10 +255,16 @@ def build_syntax(grammar: str) -> str:
     while re.search(r' [a-z0-9_]+\b', cmd):
         cmd = re.sub(r' ([a-z0-9_]+)\b', functools.partial(expand_rules, rules), cmd)
 
+    def add_indent(m: Any) -> str:
+        return '    ' + (len(m.group(1)) * '    ')
+
+    # Indent line-continuations
+    cmd = re.sub(r'^(\&+)\s*', add_indent, cmd, flags=re.M)
+
     cmd = textwrap.dedent(cmd).rstrip() + ';'
-    cmd = re.sub(r'  +', ' ', cmd)
-    cmd = re.sub(r'^ ', '    ', cmd, flags=re.M)
-    cmd = re.sub(r'\s+,\.\.\.', ',...', cmd)
+    cmd = re.sub(r'(\S)  +', r'\1 ', cmd)
+    cmd = re.sub(r'<comma>', ',', cmd)
+    cmd = re.sub(r'\s+,\s*\.\.\.', ',...', cmd)
 
     return cmd
 
@@ -399,9 +421,15 @@ def process_grammar(
     help_txt = build_help(syntax_txt, full_grammar)
     grammar = build_cmd(grammar)
 
+    # Remove line-continuations
+    grammar = re.sub(r'\n\s*&+', r'', grammar)
+
     # Make sure grouping characters all have whitespace around them
     grammar = re.sub(r' *(\[|\{|\||\}|\]) *', r' \1 ', grammar)
 
+    grammar = re.sub(r'\(', r' open_paren ', grammar)
+    grammar = re.sub(r'\)', r' close_paren ', grammar)
+
     for line in grammar.split('\n'):
         if not line.strip():
             continue
@@ -418,7 +446,7 @@ def process_grammar(
         sql = re.sub(r'\]\s+\[', r' | ', sql)
 
         # Lower-case keywords and make them case-insensitive
-        sql = re.sub(r'(\b|@+)([A-Z0-9]+)\b', lower_and_regex, sql)
+        sql = re.sub(r'(\b|@+)([A-Z0-9_]+)\b', lower_and_regex, sql)
 
         # Convert literal strings to 'qs'
         sql = re.sub(r"'[^']+'", r'qs', sql)
@@ -461,12 +489,18 @@ def process_grammar(
         sql = re.sub(r'\s+ws$', r' ws*', sql)
         sql = re.sub(r'\s+ws\s+\(', r' ws* (', sql)
         sql = re.sub(r'\)\s+ws\s+', r') ws* ', sql)
-        sql = re.sub(r'\s+ws\s+', r' ws+ ', sql)
+        sql = re.sub(r'\s+ws\s+', r' ws* ', sql)
         sql = re.sub(r'\?\s+ws\+', r'? ws*', sql)
 
         # Remove extra ws around eq
         sql = re.sub(r'ws\+\s*eq\b', r'eq', sql)
 
+        # Remove optional groupings when mandatory groupings are specified
+        sql = re.sub(r'open_paren\s+ws\*\s+open_repeats\?', r'open_paren', sql)
+        sql = re.sub(r'close_repeats\?\s+ws\*\s+close_paren', r'close_paren', sql)
+        sql = re.sub(r'open_paren\s+open_repeats\?', r'open_paren', sql)
+        sql = re.sub(r'close_repeats\?\s+close_paren', r'close_paren', sql)
+
         out.append(f'{op} = {sql}')
 
     for k, v in list(rules.items()):
@@ -548,6 +582,7 @@ class SQLHandler(NodeVisitor):
 
     def __init__(self, connection: Connection):
         self.connection = connection
+        self._handled: Set[str] = set()
 
     @classmethod
     def compile(cls, grammar: str = '') -> None:
@@ -614,12 +649,16 @@ class SQLHandler(NodeVisitor):
         )
 
         type(self).compile()
+        self._handled = set()
         try:
             params = self.visit(type(self).grammar.parse(sql))
             for k, v in params.items():
                 params[k] = self.validate_rule(k, v)
 
             res = self.run(params)
+
+            self._handled = set()
+
             if res is not None:
                 res.format_results(self.connection)
                 return res
@@ -666,16 +705,20 @@ class SQLHandler(NodeVisitor):
         """Quoted strings."""
         if node is None:
             return None
-        return node.match.group(1) or node.match.group(2) or \
-            node.match.group(3) or node.match.group(4)
+        return flatten(visited_children)[0]
+
+    def visit_compound(self, node: Node, visited_children: Iterable[Any]) -> Any:
+        """Compound name."""
+        print(visited_children)
+        return flatten(visited_children)[0]
 
     def visit_number(self, node: Node, visited_children: Iterable[Any]) -> Any:
         """Numeric value."""
-        return float(node.match.group(0))
+        return float(flatten(visited_children)[0])
 
     def visit_integer(self, node: Node, visited_children: Iterable[Any]) -> Any:
         """Integer value."""
-        return int(node.match.group(0))
+        return int(flatten(visited_children)[0])
 
     def visit_ws(self, node: Node, visited_children: Iterable[Any]) -> Any:
         """Whitespace and comments."""
@@ -804,19 +847,29 @@ class SQLHandler(NodeVisitor):
             if node.expr_name.endswith('_cmd'):
                 final = merge_dicts(flatten(visited_children)[n_keywords:])
                 for k, v in type(self).rule_info.items():
-                    if k.endswith('_cmd') or k.endswith('_'):
+                    if k.endswith('_cmd') or k.endswith('_') or k.startswith('_'):
                         continue
-                    if k not in final:
+                    if k not in final and k not in self._handled:
                         final[k] = BUILTIN_DEFAULTS.get(k, v['default'])
                 return final
 
             # Filter out stray empty strings
             out = [x for x in flatten(visited_children)[n_keywords:] if x]
 
-            if repeats or len(out) > 1:
-                return {node.expr_name: out}
+            # Remove underscore prefixes from rule name
+            key_name = re.sub(r'^_+', r'', node.expr_name)
 
-            return {node.expr_name: out[0] if out else True}
+            if repeats or len(out) > 1:
+                self._handled.add(node.expr_name)
+                # If all outputs are dicts, merge them
+                if len(out) > 1 and not repeats:
+                    is_dicts = [x for x in out if isinstance(x, dict)]
+                    if len(is_dicts) == len(out):
+                        return {key_name: merge_dicts(out)}
+                return {key_name: out}
+
+            self._handled.add(node.expr_name)
+            return {key_name: out[0] if out else True}
 
         if hasattr(node, 'match'):
             if not visited_children and not node.match.groups():

singlestoredb/fusion/handlers/export.py

@@ -0,0 +1,237 @@
+#!/usr/bin/env python3
+import json
+from typing import Any
+from typing import Dict
+from typing import Optional
+
+from .. import result
+from ...management.export import ExportService
+from ...management.export import ExportStatus
+from ..handler import SQLHandler
+from ..result import FusionSQLResult
+from .utils import get_workspace_group
+
+
+class CreateClusterIdentity(SQLHandler):
+    """
+    CREATE CLUSTER IDENTITY
+        catalog
+        storage
+    ;
+
+    # Catolog
+    catalog = CATALOG { _catalog_config | _catalog_creds }
+    _catalog_config = CONFIG '<catalog-config>'
+    _catalog_creds = CREDENTIALS '<catalog-creds>'
+
+    # Storage
+    storage = LINK { _link_config | _link_creds }
+    _link_config = S3 CONFIG '<link-config>'
+    _link_creds = CREDENTIALS '<link-creds>'
+
+    Description
+    -----------
+    Create a cluster identity for allowing the export service to access
+    external cloud resources.
+
+    Arguments
+    ---------
+    * ``<catalog-config>`` and ``<catalog-creds>``: Catalog configuration
+      and credentials in JSON format.
+    * ``<link-config>`` and ``<link-creds>``: Storage link configuration
+      and credentials in JSON format.
+
+    Remarks
+    -------
+    * ``FROM <table>`` specifies the SingleStore table to export. The same name will
+      be used for the exported table.
+    * ``CATALOG`` specifies the details of the catalog to connect to.
+    * ``LINK`` specifies the details of the data storage to connect to.
+
+    Example
+    -------
+    The following statement creates a cluster identity for the catalog
+    and link::
+
+        CREATE CLUSTER IDENTITY
+            CATALOG CONFIG '{
+                "type": "GLUE",
+                "table_format": "ICEBERG",
+                "id": "13983498723498",
+                "region": "us-east-1"
+            }'
+            LINK S3 CONFIG '{
+                "region": "us-east-1",
+                "endpoint_url": "s3://bucket-name"
+
+            }'
+        ;
+
+    """
+
+    def run(self, params: Dict[str, Any]) -> Optional[FusionSQLResult]:
+        # Catalog
+        catalog_config = json.loads(params['catalog'].get('catalog_config', '{}') or '{}')
+        catalog_creds = json.loads(params['catalog'].get('catalog_creds', '{}') or '{}')
+
+        # Storage
+        storage_config = json.loads(params['storage'].get('link_config', '{}') or '{}')
+        storage_creds = json.loads(params['storage'].get('link_creds', '{}') or '{}')
+
+        wsg = get_workspace_group({})
+
+        if wsg._manager is None:
+            raise TypeError('no workspace manager is associated with workspace group')
+
+        out = ExportService(
+            wsg,
+            'none',
+            'none',
+            dict(**catalog_config, **catalog_creds),
+            dict(**storage_config, **storage_creds),
+            columns=None,
+        ).create_cluster_identity()
+
+        res = FusionSQLResult()
+        res.add_field('Identity', result.STRING)
+        res.set_rows([(out['identity'],)])
+
+        return res
+
+
+CreateClusterIdentity.register(overwrite=True)
+
+
+class CreateExport(SQLHandler):
+    """
+    START EXPORT
+        from_table
+        catalog
+        storage
+    ;
+
+    # From table
+    from_table = FROM <table>
+
+    # Catolog
+    catalog = CATALOG [ _catalog_config ] [ _catalog_creds ]
+    _catalog_config = CONFIG '<catalog-config>'
+    _catalog_creds = CREDENTIALS '<catalog-creds>'
+
+    # Storage
+    storage = LINK [ _link_config ] [ _link_creds ]
+    _link_config = S3 CONFIG '<link-config>'
+    _link_creds = CREDENTIALS '<link-creds>'
+
+    Description
+    -----------
+    Create an export configuration.
+
+    Arguments
+    ---------
+    * ``<catalog-config>`` and ``<catalog-creds>``: The catalog configuration.
+    * ``<link-config>`` and ``<link-creds>``: The storage link configuration.
+
+    Remarks
+    -------
+    * ``FROM <table>`` specifies the SingleStore table to export. The same name will
+      be used for the exported table.
+    * ``CATALOG`` specifies the details of the catalog to connect to.
+    * ``LINK`` specifies the details of the data storage to connect to.
+
+    Examples
+    --------
+    The following statement starts an export operation with the given
+    catalog and link configurations. The source table to export is
+    named "customer_data"::
+
+        START EXPORT FROM customer_data
+            CATALOG CONFIG '{
+                "type": "GLUE",
+                "table_format": "ICEBERG",
+                "id": "13983498723498",
+                "region": "us-east-1"
+            }'
+            LINK S3 CONFIG '{
+                "region": "us-east-1",
+                "endpoint_url": "s3://bucket-name"
+
+            }'
+        ;
+
+    """  # noqa
+
+    def run(self, params: Dict[str, Any]) -> Optional[FusionSQLResult]:
+        # From table
+        if isinstance(params['from_table'], str):
+            from_database = None
+            from_table = params['from_table']
+        else:
+            from_database, from_table = params['from_table']
+
+        # Catalog
+        catalog_config = json.loads(params['catalog'].get('catalog_config', '{}') or '{}')
+        catalog_creds = json.loads(params['catalog'].get('catalog_creds', '{}') or '{}')
+
+        # Storage
+        storage_config = json.loads(params['storage'].get('link_config', '{}') or '{}')
+        storage_creds = json.loads(params['storage'].get('link_creds', '{}') or '{}')
+
+        wsg = get_workspace_group({})
+
+        if from_database is None:
+            raise ValueError('database name must be specified for source table')
+
+        if wsg._manager is None:
+            raise TypeError('no workspace manager is associated with workspace group')
+
+        out = ExportService(
+            wsg,
+            from_database,
+            from_table,
+            dict(**catalog_config, **catalog_creds),
+            dict(**storage_config, **storage_creds),
+            columns=None,
+        ).start()
+
+        res = FusionSQLResult()
+        res.add_field('ExportID', result.STRING)
+        res.set_rows([(out.export_id,)])
+
+        return res
+
+
+CreateExport.register(overwrite=True)
+
+
+class ShowExport(SQLHandler):
+    """
+    SHOW EXPORT export_id;
+
+    # ID of export
+    export_id = '<export-id>'
+
+    """
+
+    def run(self, params: Dict[str, Any]) -> Optional[FusionSQLResult]:
+        wsg = get_workspace_group({})
+        out = ExportStatus(params['export_id'], wsg)
+
+        status = out._info()
+
+        res = FusionSQLResult()
+        res.add_field('ExportID', result.STRING)
+        res.add_field('Status', result.STRING)
+        res.add_field('Message', result.STRING)
+        res.set_rows([
+            (
+                params['export_id'],
+                status.get('status', 'Unknown'),
+                status.get('statusMsg', ''),
+            ),
+        ])
+
+        return res
+
+
+ShowExport.register(overwrite=True)