singlestoredb 1.7.2__tar.gz → 1.8.0__tar.gz

{singlestoredb-1.7.2/singlestoredb.egg-info → singlestoredb-1.8.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: singlestoredb
-Version: 1.7.2
+Version: 1.8.0
 Summary: Interface to the SingleStoreDB database and workspace management APIs
 Home-page: https://github.com/singlestore-labs/singlestoredb-python
 Author: SingleStore

{singlestoredb-1.7.2 → singlestoredb-1.8.0}/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = singlestoredb
-version = 1.7.2
+version = 1.8.0
 description = Interface to the SingleStoreDB database and workspace management APIs
 long_description = file: README.md
 long_description_content_type = text/markdown
@@ -28,6 +28,7 @@ install_requires =
 	wheel
 	tomli>=1.1.0;python_version < '3.11'
 python_requires = >=3.8
+include_package_data = True
 tests_require = 
 	coverage
 	pytest
@@ -66,6 +67,7 @@ sqlalchemy =
 
 [options.package_data]
 * = 
+	*.typed
 	*.sql
 	*.csv
 

{singlestoredb-1.7.2 → singlestoredb-1.8.0}/singlestoredb/__init__.py

@@ -13,7 +13,7 @@ Examples
 
 """
 
-__version__ = '1.7.2'
+__version__ = '1.8.0'
 
 from typing import Any
 

{singlestoredb-1.7.2 → singlestoredb-1.8.0}/singlestoredb/fusion/handler.py

@@ -11,6 +11,7 @@ from typing import Dict
 from typing import Iterable
 from typing import List
 from typing import Optional
+from typing import Set
 from typing import Tuple
 
 from parsimonious import Grammar
@@ -23,9 +24,9 @@ from ..connection import Connection
 
 CORE_GRAMMAR = r'''
     ws = ~r"(\s+|(\s*/\*.*\*/\s*)+)"
-    qs = ~r"\"([^\"]*)\"|'([^\']*)'|`([^\`]*)`|([A-Za-z0-9_\-\.]+)"
-    number = ~r"[-+]?(\d*\.)?\d+(e[-+]?\d+)?"i
-    integer = ~r"-?\d+"
+    qs = ~r"\"([^\"]*)\"|'([^\']*)'|([A-Za-z0-9_\-\.]+)|`([^\`]+)`" ws*
+    number = ~r"[-+]?(\d*\.)?\d+(e[-+]?\d+)?"i ws*
+    integer = ~r"-?\d+" ws*
     comma = ws* "," ws*
     eq = ws* "=" ws*
     open_paren = ws* "(" ws*
@@ -33,6 +34,10 @@ CORE_GRAMMAR = r'''
     open_repeats = ws* ~r"[\(\[\{]" ws*
     close_repeats = ws* ~r"[\)\]\}]" ws*
     select = ~r"SELECT"i ws+ ~r".+" ws*
+    table = ~r"(?:([A-Za-z0-9_\-]+)|`([^\`]+)`)(?:\.(?:([A-Za-z0-9_\-]+)|`([^\`]+)`))?" ws*
+    column = ~r"(?:([A-Za-z0-9_\-]+)|`([^\`]+)`)(?:\.(?:([A-Za-z0-9_\-]+)|`([^\`]+)`))?" ws*
+    link_name = ~r"(?:([A-Za-z0-9_\-]+)|`([^\`]+)`)(?:\.(?:([A-Za-z0-9_\-]+)|`([^\`]+)`))?" ws*
+    catalog_name = ~r"(?:([A-Za-z0-9_\-]+)|`([^\`]+)`)(?:\.(?:([A-Za-z0-9_\-]+)|`([^\`]+)`))?" ws*
 
     json = ws* json_object ws*
     json_object = ~r"{\s*" json_members? ~r"\s*}"
@@ -65,6 +70,10 @@ BUILTINS = {
     '<integer>': '',
     '<number>': '',
     '<json>': '',
+    '<table>': '',
+    '<column>': '',
+    '<catalog-name>': '',
+    '<link-name>': '',
 }
 
 BUILTIN_DEFAULTS = {  # type: ignore
@@ -226,9 +235,13 @@ def build_syntax(grammar: str) -> str:
     # Split on ';' on a line by itself
     cmd, end = grammar.split(';', 1)
 
-    rules = {}
+    name = ''
+    rules: Dict[str, Any] = {}
     for line in end.split('\n'):
         line = line.strip()
+        if line.startswith('&'):
+            rules[name] += '\n' + line
+            continue
         if not line:
             continue
         name, value = line.split('=', 1)
@@ -239,10 +252,16 @@ def build_syntax(grammar: str) -> str:
     while re.search(r' [a-z0-9_]+\b', cmd):
         cmd = re.sub(r' ([a-z0-9_]+)\b', functools.partial(expand_rules, rules), cmd)
 
+    def add_indent(m: Any) -> str:
+        return '    ' + (len(m.group(1)) * '    ')
+
+    # Indent line-continuations
+    cmd = re.sub(r'^(\&+)\s*', add_indent, cmd, flags=re.M)
+
     cmd = textwrap.dedent(cmd).rstrip() + ';'
-    cmd = re.sub(r'  +', ' ', cmd)
-    cmd = re.sub(r'^ ', '    ', cmd, flags=re.M)
-    cmd = re.sub(r'\s+,\.\.\.', ',...', cmd)
+    cmd = re.sub(r'(\S)  +', r'\1 ', cmd)
+    cmd = re.sub(r'<comma>', ',', cmd)
+    cmd = re.sub(r'\s+,\s*\.\.\.', ',...', cmd)
 
     return cmd
 
@@ -399,9 +418,15 @@ def process_grammar(
     help_txt = build_help(syntax_txt, full_grammar)
     grammar = build_cmd(grammar)
 
+    # Remove line-continuations
+    grammar = re.sub(r'\n\s*&+', r'', grammar)
+
     # Make sure grouping characters all have whitespace around them
     grammar = re.sub(r' *(\[|\{|\||\}|\]) *', r' \1 ', grammar)
 
+    grammar = re.sub(r'\(', r' open_paren ', grammar)
+    grammar = re.sub(r'\)', r' close_paren ', grammar)
+
     for line in grammar.split('\n'):
         if not line.strip():
             continue
@@ -418,7 +443,7 @@ def process_grammar(
         sql = re.sub(r'\]\s+\[', r' | ', sql)
 
         # Lower-case keywords and make them case-insensitive
-        sql = re.sub(r'(\b|@+)([A-Z0-9]+)\b', lower_and_regex, sql)
+        sql = re.sub(r'(\b|@+)([A-Z0-9_]+)\b', lower_and_regex, sql)
 
         # Convert literal strings to 'qs'
         sql = re.sub(r"'[^']+'", r'qs', sql)
@@ -461,12 +486,18 @@ def process_grammar(
         sql = re.sub(r'\s+ws$', r' ws*', sql)
         sql = re.sub(r'\s+ws\s+\(', r' ws* (', sql)
         sql = re.sub(r'\)\s+ws\s+', r') ws* ', sql)
-        sql = re.sub(r'\s+ws\s+', r' ws+ ', sql)
+        sql = re.sub(r'\s+ws\s+', r' ws* ', sql)
         sql = re.sub(r'\?\s+ws\+', r'? ws*', sql)
 
         # Remove extra ws around eq
         sql = re.sub(r'ws\+\s*eq\b', r'eq', sql)
 
+        # Remove optional groupings when mandatory groupings are specified
+        sql = re.sub(r'open_paren\s+ws\*\s+open_repeats\?', r'open_paren', sql)
+        sql = re.sub(r'close_repeats\?\s+ws\*\s+close_paren', r'close_paren', sql)
+        sql = re.sub(r'open_paren\s+open_repeats\?', r'open_paren', sql)
+        sql = re.sub(r'close_repeats\?\s+close_paren', r'close_paren', sql)
+
         out.append(f'{op} = {sql}')
 
     for k, v in list(rules.items()):
@@ -548,6 +579,7 @@ class SQLHandler(NodeVisitor):
 
     def __init__(self, connection: Connection):
         self.connection = connection
+        self._handled: Set[str] = set()
 
     @classmethod
     def compile(cls, grammar: str = '') -> None:
@@ -614,12 +646,16 @@ class SQLHandler(NodeVisitor):
         )
 
         type(self).compile()
+        self._handled = set()
         try:
             params = self.visit(type(self).grammar.parse(sql))
             for k, v in params.items():
                 params[k] = self.validate_rule(k, v)
 
             res = self.run(params)
+
+            self._handled = set()
+
             if res is not None:
                 res.format_results(self.connection)
                 return res
@@ -666,16 +702,20 @@ class SQLHandler(NodeVisitor):
         """Quoted strings."""
         if node is None:
             return None
-        return node.match.group(1) or node.match.group(2) or \
-            node.match.group(3) or node.match.group(4)
+        return flatten(visited_children)[0]
+
+    def visit_compound(self, node: Node, visited_children: Iterable[Any]) -> Any:
+        """Compound name."""
+        print(visited_children)
+        return flatten(visited_children)[0]
 
     def visit_number(self, node: Node, visited_children: Iterable[Any]) -> Any:
         """Numeric value."""
-        return float(node.match.group(0))
+        return float(flatten(visited_children)[0])
 
     def visit_integer(self, node: Node, visited_children: Iterable[Any]) -> Any:
         """Integer value."""
-        return int(node.match.group(0))
+        return int(flatten(visited_children)[0])
 
     def visit_ws(self, node: Node, visited_children: Iterable[Any]) -> Any:
         """Whitespace and comments."""
@@ -804,19 +844,29 @@ class SQLHandler(NodeVisitor):
             if node.expr_name.endswith('_cmd'):
                 final = merge_dicts(flatten(visited_children)[n_keywords:])
                 for k, v in type(self).rule_info.items():
-                    if k.endswith('_cmd') or k.endswith('_'):
+                    if k.endswith('_cmd') or k.endswith('_') or k.startswith('_'):
                         continue
-                    if k not in final:
+                    if k not in final and k not in self._handled:
                         final[k] = BUILTIN_DEFAULTS.get(k, v['default'])
                 return final
 
             # Filter out stray empty strings
             out = [x for x in flatten(visited_children)[n_keywords:] if x]
 
-            if repeats or len(out) > 1:
-                return {node.expr_name: out}
+            # Remove underscore prefixes from rule name
+            key_name = re.sub(r'^_+', r'', node.expr_name)
 
-            return {node.expr_name: out[0] if out else True}
+            if repeats or len(out) > 1:
+                self._handled.add(node.expr_name)
+                # If all outputs are dicts, merge them
+                if len(out) > 1 and not repeats:
+                    is_dicts = [x for x in out if isinstance(x, dict)]
+                    if len(is_dicts) == len(out):
+                        return {key_name: merge_dicts(out)}
+                return {key_name: out}
+
+            self._handled.add(node.expr_name)
+            return {key_name: out[0] if out else True}
 
         if hasattr(node, 'match'):
             if not visited_children and not node.match.groups():

singlestoredb-1.8.0/singlestoredb/fusion/handlers/export.py

@@ -0,0 +1,245 @@
+#!/usr/bin/env python3
+import json
+from typing import Any
+from typing import Dict
+from typing import Optional
+
+from .. import result
+from ...management.export import Catalog
+from ...management.export import ExportService
+from ...management.export import ExportStatus
+from ...management.export import Link
+from ..handler import SQLHandler
+from ..result import FusionSQLResult
+from .utils import get_workspace_group
+
+
+class CreateClusterIdentity(SQLHandler):
+    """
+    CREATE CLUSTER IDENTITY
+        catalog
+        storage
+    ;
+
+    # Catolog
+    catalog = CATALOG { _catalog_config | _catalog_creds }
+    _catalog_config = CONFIG '<catalog-config>'
+    _catalog_creds = CREDENTIALS '<catalog-creds>'
+
+    # Storage
+    storage = LINK { _link_config | _link_creds }
+    _link_config = S3 CONFIG '<link-config>'
+    _link_creds = CREDENTIALS '<link-creds>'
+
+    Description
+    -----------
+    Create a cluster identity for allowing the export service to access
+    external cloud resources.
+
+    Arguments
+    ---------
+    * ``<catalog-config>`` and ``<catalog-creds>``: Catalog configuration
+      and credentials in JSON format.
+    * ``<link-config>`` and ``<link-creds>``: Storage link configuration
+      and credentials in JSON format.
+
+    Remarks
+    -------
+    * ``FROM <table>`` specifies the SingleStore table to export. The same name will
+      be used for the exported table.
+    * ``CATALOG`` specifies the details of the catalog to connect to.
+    * ``LINK`` specifies the details of the data storage to connect to.
+
+    Example
+    -------
+    The following statement creates a cluster identity for the catalog
+    and link::
+
+        CREATE CLUSTER IDENTITY
+            CATALOG CONFIG '{
+                "type": "GLUE",
+                "table_format": "ICEBERG",
+                "id": "13983498723498",
+                "region": "us-east-1"
+            }'
+            LINK S3 CONFIG '{
+                "region": "us-east-1",
+                "endpoint_url": "s3://bucket-name"
+
+            }'
+        ;
+
+    """
+
+    _enabled = False
+
+    def run(self, params: Dict[str, Any]) -> Optional[FusionSQLResult]:
+        # Catalog
+        catalog_config = json.loads(params['catalog'].get('catalog_config', '{}') or '{}')
+        catalog_creds = json.loads(params['catalog'].get('catalog_creds', '{}') or '{}')
+
+        # Storage
+        storage_config = json.loads(params['storage'].get('link_config', '{}') or '{}')
+        storage_creds = json.loads(params['storage'].get('link_creds', '{}') or '{}')
+
+        wsg = get_workspace_group({})
+
+        if wsg._manager is None:
+            raise TypeError('no workspace manager is associated with workspace group')
+
+        out = ExportService(
+            wsg,
+            'none',
+            'none',
+            Catalog.from_config_and_creds(catalog_config, catalog_creds, wsg._manager),
+            Link.from_config_and_creds('S3', storage_config, storage_creds, wsg._manager),
+            columns=None,
+        ).create_cluster_identity()
+
+        res = FusionSQLResult()
+        res.add_field('RoleARN', result.STRING)
+        res.set_rows([(out['roleARN'],)])
+
+        return res
+
+
+CreateClusterIdentity.register(overwrite=True)
+
+
+class CreateExport(SQLHandler):
+    """
+    START EXPORT
+        from_table
+        catalog
+        storage
+    ;
+
+    # From table
+    from_table = FROM <table>
+
+    # Catolog
+    catalog = CATALOG [ _catalog_config ] [ _catalog_creds ]
+    _catalog_config = CONFIG '<catalog-config>'
+    _catalog_creds = CREDENTIALS '<catalog-creds>'
+
+    # Storage
+    storage = LINK [ _link_config ] [ _link_creds ]
+    _link_config = S3 CONFIG '<link-config>'
+    _link_creds = CREDENTIALS '<link-creds>'
+
+    Description
+    -----------
+    Create an export configuration.
+
+    Arguments
+    ---------
+    * ``<catalog-config>`` and ``<catalog-creds>``: The catalog configuration.
+    * ``<link-config>`` and ``<link-creds>``: The storage link configuration.
+
+    Remarks
+    -------
+    * ``FROM <table>`` specifies the SingleStore table to export. The same name will
+      be used for the exported table.
+    * ``CATALOG`` specifies the details of the catalog to connect to.
+    * ``LINK`` specifies the details of the data storage to connect to.
+
+    Examples
+    --------
+    The following statement starts an export operation with the given
+    catalog and link configurations. The source table to export is
+    named "customer_data"::
+
+        START EXPORT FROM customer_data
+            CATALOG CONFIG '{
+                "type": "GLUE",
+                "table_format": "ICEBERG",
+                "id": "13983498723498",
+                "region": "us-east-1"
+            }'
+            LINK S3 CONFIG '{
+                "region": "us-east-1",
+                "endpoint_url": "s3://bucket-name"
+
+            }'
+        ;
+
+    """  # noqa
+
+    _enabled = False
+
+    def run(self, params: Dict[str, Any]) -> Optional[FusionSQLResult]:
+        # From table
+        if isinstance(params['from_table'], str):
+            from_database = None
+            from_table = params['from_table']
+        else:
+            from_database, from_table = params['from_table']
+
+        # Catalog
+        catalog_config = json.loads(params['catalog'].get('catalog_config', '{}') or '{}')
+        catalog_creds = json.loads(params['catalog'].get('catalog_creds', '{}') or '{}')
+
+        # Storage
+        storage_config = json.loads(params['storage'].get('link_config', '{}') or '{}')
+        storage_creds = json.loads(params['storage'].get('link_creds', '{}') or '{}')
+
+        wsg = get_workspace_group({})
+
+        if from_database is None:
+            raise ValueError('database name must be specified for source table')
+
+        if wsg._manager is None:
+            raise TypeError('no workspace manager is associated with workspace group')
+
+        out = ExportService(
+            wsg,
+            from_database,
+            from_table,
+            Catalog.from_config_and_creds(catalog_config, catalog_creds, wsg._manager),
+            Link.from_config_and_creds('S3', storage_config, storage_creds, wsg._manager),
+            columns=None,
+        ).start()
+
+        res = FusionSQLResult()
+        res.add_field('ExportID', result.STRING)
+        res.set_rows([(out.export_id,)])
+
+        return res
+
+
+CreateExport.register(overwrite=True)
+
+
+class ShowExport(SQLHandler):
+    """
+    SHOW EXPORT export_id;
+
+    # ID of export
+    export_id = '<export-id>'
+
+    """
+
+    _enabled = False
+
+    def run(self, params: Dict[str, Any]) -> Optional[FusionSQLResult]:
+        wsg = get_workspace_group({})
+        out = ExportStatus(params['export_id'], wsg)
+
+        status = out._info()
+
+        res = FusionSQLResult()
+        res.add_field('ExportID', result.STRING)
+        res.add_field('Status', result.STRING)
+        res.add_field('Message', result.STRING)
+        res.set_rows([
+            (
+                params['export_id'],
+                status.get('status', 'Unknown'),
+                status.get('statusMsg', ''),
+            ),
+        ])
+
+        return res
+
+
+ShowExport.register(overwrite=True)

{singlestoredb-1.7.2 → singlestoredb-1.8.0}/singlestoredb/fusion/handlers/job.py

@@ -122,8 +122,6 @@ class ScheduleJobHandler(SQLHandler):
         ;
     """
 
-    _enabled = False
-
     def run(self, params: Dict[str, Any]) -> Optional[FusionSQLResult]:
         res = FusionSQLResult()
         res.add_field('JobID', result.STRING)
@@ -138,8 +136,8 @@ class ScheduleJobHandler(SQLHandler):
 
         execution_interval_in_mins = None
         if params.get('execute_every'):
-            execution_interval_in_mins = params['execute_every'][0]['interval']
-            time_unit = params['execute_every'][-1]['time_unit'].upper()
+            execution_interval_in_mins = params['execute_every']['interval']
+            time_unit = params['execute_every']['time_unit'].upper()
             if time_unit == 'MINUTES':
                 pass
             elif time_unit == 'HOURS':
@@ -224,8 +222,6 @@ class RunJobHandler(SQLHandler):
 
     """
 
-    _enabled = False
-
     def run(self, params: Dict[str, Any]) -> Optional[FusionSQLResult]:
         res = FusionSQLResult()
         res.add_field('JobID', result.STRING)
@@ -288,8 +284,6 @@ class WaitOnJobsHandler(SQLHandler):
 
     """
 
-    _enabled = False
-
     def run(self, params: Dict[str, Any]) -> Optional[FusionSQLResult]:
         res = FusionSQLResult()
         res.add_field('Success', result.BOOL)
@@ -298,8 +292,8 @@ class WaitOnJobsHandler(SQLHandler):
 
         timeout_in_secs = None
         if params.get('with_timeout'):
-            timeout_in_secs = params['with_timeout'][0]['time']
-            time_unit = params['with_timeout'][-1]['time_unit'].upper()
+            timeout_in_secs = params['with_timeout']['time']
+            time_unit = params['with_timeout']['time_unit'].upper()
             if time_unit == 'SECONDS':
                 pass
             elif time_unit == 'MINUTES':
@@ -359,8 +353,6 @@ class ShowJobsHandler(SQLHandler):
 
     """
 
-    _enabled = False
-
     def run(self, params: Dict[str, Any]) -> Optional[FusionSQLResult]:
         res = FusionSQLResult()
         res.add_field('JobID', result.STRING)
@@ -492,8 +484,6 @@ class ShowJobExecutionsHandler(SQLHandler):
           EXTENDED;
     """
 
-    _enabled = False
-
     def run(self, params: Dict[str, Any]) -> Optional[FusionSQLResult]:
         res = FusionSQLResult()
         res.add_field('ExecutionID', result.STRING)
@@ -564,8 +554,6 @@ class ShowJobParametersHandler(SQLHandler):
         SHOW JOB PARAMETERS FOR 'job1';
     """
 
-    _enabled = False
-
     def run(self, params: Dict[str, Any]) -> Optional[FusionSQLResult]:
         res = FusionSQLResult()
         res.add_field('Name', result.STRING)
@@ -606,8 +594,6 @@ class ShowJobRuntimesHandler(SQLHandler):
         SHOW JOB RUNTIMES;
     """
 
-    _enabled = False
-
     def run(self, params: Dict[str, Any]) -> Optional[FusionSQLResult]:
         res = FusionSQLResult()
         res.add_field('Name', result.STRING)
@@ -653,8 +639,6 @@ class DropJobHandler(SQLHandler):
         DROP JOBS 'job1', 'job2';
     """
 
-    _enabled = False
-
     def run(self, params: Dict[str, Any]) -> Optional[FusionSQLResult]:
         res = FusionSQLResult()
         res.add_field('JobID', result.STRING)