singlestoredb 1.4.3__cp38-abi3-win32.whl → 1.6.0__cp38-abi3-win32.whl

singlestoredb/management/organization.py

@@ -7,6 +7,7 @@ from typing import Optional
 from typing import Union
 
 from ..exceptions import ManagementError
+from .job import JobsManager
 from .manager import Manager
 from .utils import vars_to_str
 
@@ -190,3 +191,19 @@ class Organization(object):
         )
         out._manager = manager
         return out
+
+    @property
+    def jobs(self) -> JobsManager:
+        """
+        Retrieve a SingleStoreDB scheduled job manager.
+
+        Parameters
+        ----------
+        manager : WorkspaceManager, optional
+            The WorkspaceManager the JobsManager belongs to
+
+        Returns
+        -------
+        :class:`JobsManager`
+        """
+        return JobsManager(self._manager)

singlestoredb/management/utils.py

@@ -215,6 +215,26 @@ def get_token() -> Optional[str]:
     return None
 
 
+def get_cluster_id() -> Optional[str]:
+    """Return the cluster id for the current token or environment."""
+    return os.environ.get('SINGLESTOREDB_CLUSTER') or None
+
+
+def get_workspace_id() -> Optional[str]:
+    """Return the workspace id for the current token or environment."""
+    return os.environ.get('SINGLESTOREDB_WORKSPACE') or None
+
+
+def get_virtual_workspace_id() -> Optional[str]:
+    """Return the virtual workspace id for the current token or environment."""
+    return os.environ.get('SINGLESTOREDB_VIRTUAL_WORKSPACE') or None
+
+
+def get_database_name() -> Optional[str]:
+    """Return the default database name for the current token or environment."""
+    return os.environ.get('SINGLESTOREDB_DEFAULT_DATABASE') or None
+
+
 def enable_http_tracing() -> None:
     """Enable tracing of HTTP requests."""
     import logging
@@ -246,7 +266,33 @@ def to_datetime(
     out = converters.datetime_fromisoformat(obj)
     if isinstance(out, str):
         return None
-    if isinstance(out, datetime.date):
+    if isinstance(out, datetime.date) and not isinstance(out, datetime.datetime):
+        return datetime.datetime(out.year, out.month, out.day)
+    return out
+
+
+def to_datetime_strict(
+    obj: Optional[Union[str, datetime.datetime]],
+) -> datetime.datetime:
+    """Convert string to datetime."""
+    if not obj:
+        raise TypeError('not possible to convert None to datetime')
+    if isinstance(obj, datetime.datetime):
+        return obj
+    if obj == '0001-01-01T00:00:00Z':
+        raise ValueError('not possible to convert 0001-01-01T00:00:00Z to datetime')
+    obj = obj.replace('Z', '')
+    # Fix datetimes with truncated zeros
+    if '.' in obj:
+        obj, micros = obj.split('.', 1)
+        micros = micros + '0' * (6 - len(micros))
+        obj = obj + '.' + micros
+    out = converters.datetime_fromisoformat(obj)
+    if not out:
+        raise TypeError('not possible to convert None to datetime')
+    if isinstance(out, str):
+        raise ValueError('value cannot be str')
+    if isinstance(out, datetime.date) and not isinstance(out, datetime.datetime):
         return datetime.datetime(out.year, out.month, out.day)
     return out
 

singlestoredb/mysql/connection.py

@@ -280,6 +280,8 @@ class Connection(BaseConnection):
         Should the connection track the SINGLESTOREDB_URL environment variable?
     enable_extended_data_types : bool, optional
         Should extended data types (BSON, vector) be enabled?
+    vector_data_format : str, optional
+        Specify the data type of vector values: json or binary
 
     See `Connection <https://www.python.org/dev/peps/pep-0249/#connection-objects>`_
     in the specification.
@@ -350,6 +352,7 @@ class Connection(BaseConnection):
         encoding_errors='strict',
         track_env=False,
         enable_extended_data_types=True,
+        vector_data_format='binary',
     ):
         BaseConnection.__init__(**dict(locals()))
 
@@ -634,6 +637,13 @@ class Connection(BaseConnection):
         self._in_sync = False
         self._track_env = bool(track_env) or self.host == 'singlestore.com'
         self._enable_extended_data_types = enable_extended_data_types
+        if vector_data_format.lower() in ['json', 'binary']:
+            self._vector_data_format = vector_data_format
+        else:
+            raise ValueError(
+                'unknown value for vector_data_format, '
+                f'expecting "json" or "binary": {vector_data_format}',
+            )
         self._connection_info = {}
         events.subscribe(self._handle_event)
 
@@ -1117,6 +1127,15 @@ class Connection(BaseConnection):
                     pass
                 c.close()
 
+            if self._vector_data_format:
+                c = self.cursor()
+                try:
+                    val = self._vector_data_format
+                    c.execute(f'SET @@SESSION.vector_type_project_format={val}')
+                except self.OperationalError:
+                    pass
+                c.close()
+
             if self.init_command is not None:
                 c = self.cursor()
                 c.execute(self.init_command)

singlestoredb/notebook/_portal.py

@@ -9,6 +9,7 @@ from typing import Callable
 from typing import Dict
 from typing import List
 from typing import Optional
+from typing import Tuple
 
 from . import _objects as obj
 from ..management import workspace as mgr
@@ -187,6 +188,37 @@ class Portal(object):
             timeout_message='timeout waiting for workspace update',
         )
 
+    deployment = workspace
+
+    @property
+    def connection(self) -> Tuple[obj.Workspace, Optional[str]]:
+        """Workspace and default database name."""
+        return self.workspace, self.default_database
+
+    @connection.setter
+    def connection(self, workspace_and_default_database: Tuple[str, str]) -> None:
+        """Set workspace and default database name."""
+        name_or_id, default_database = workspace_and_default_database
+        if re.match(
+            r'[a-z0-9]{8}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12}',
+            name_or_id, flags=re.I,
+        ):
+            w = mgr.get_workspace(name_or_id)
+        else:
+            w = mgr.get_workspace_group(self.workspace_group_id).workspaces[name_or_id]
+
+        if w.state and w.state.lower() not in ['active', 'resumed']:
+            raise RuntimeError('workspace is not active')
+
+        id = w.id
+
+        self._call_javascript(
+            'changeConnection', [id, default_database],
+            wait_on_condition=lambda: self.workspace_id == id and
+            self.default_database == default_database,  # type: ignore
+            timeout_message='timeout waiting for workspace update',
+        )
+
     @property
     def cluster_id(self) -> Optional[str]:
         """Cluster ID."""

singlestoredb/tests/test_fusion.py

@@ -91,7 +91,7 @@ class TestFusion(unittest.TestCase):
 
 
 @pytest.mark.management
-class TestManagementAPIFusion(unittest.TestCase):
+class TestWorkspaceFusion(unittest.TestCase):
 
     id: str = secrets.token_hex(8)
     dbname: str = ''
@@ -463,3 +463,252 @@ class TestManagementAPIFusion(unittest.TestCase):
                 mgr.workspace_groups[wg_name].terminate(force=True)
             except Exception:
                 pass
+
+
+@pytest.mark.management
+class TestJobsFusion(unittest.TestCase):
+
+    id: str = secrets.token_hex(8)
+    notebook_name: str = 'Scheduling Test.ipynb'
+    dbname: str = ''
+    dbexisted: bool = False
+    manager: None
+    workspace_group: None
+    workspace: None
+    job_ids = []
+
+    @classmethod
+    def setUpClass(cls):
+        sql_file = os.path.join(os.path.dirname(__file__), 'test.sql')
+        cls.dbname, cls.dbexisted = utils.load_sql(sql_file)
+        cls.manager = s2.manage_workspaces()
+        us_regions = [x for x in cls.manager.regions if x.name.startswith('US')]
+        cls.workspace_group = cls.manager.create_workspace_group(
+            f'Jobs Fusion Testing {cls.id}',
+            region=random.choice(us_regions),
+            firewall_ranges=[],
+        )
+        cls.workspace = cls.workspace_group.create_workspace(
+                f'jobs-test-{cls.id}',
+                wait_on_active=True,
+        )
+        os.environ['SINGLESTOREDB_DEFAULT_DATABASE'] = cls.dbname
+        os.environ['SINGLESTOREDB_WORKSPACE'] = cls.workspace.id
+
+    @classmethod
+    def tearDownClass(cls):
+        for job_id in cls.job_ids:
+            cls.manager.organizations.current.jobs.delete(job_id)
+        if cls.workspace_group is not None:
+            cls.workspace_group.terminate(force=True)
+        cls.manager = None
+        cls.workspace_group = None
+        cls.workspace = None
+        if os.environ.get('SINGLESTOREDB_WORKSPACE', None) is not None:
+            del os.environ['SINGLESTOREDB_WORKSPACE']
+        if os.environ.get('SINGLESTOREDB_DEFAULT_DATABASE', None) is not None:
+            del os.environ['SINGLESTOREDB_DEFAULT_DATABASE']
+
+    def setUp(self):
+        self.enabled = os.environ.get('SINGLESTOREDB_FUSION_ENABLED')
+        os.environ['SINGLESTOREDB_FUSION_ENABLED'] = '1'
+        self.conn = s2.connect(database=type(self).dbname, local_infile=True)
+        self.cur = self.conn.cursor()
+
+    def tearDown(self):
+        if self.enabled:
+            os.environ['SINGLESTOREDB_FUSION_ENABLED'] = self.enabled
+        else:
+            del os.environ['SINGLESTOREDB_FUSION_ENABLED']
+
+        try:
+            if self.cur is not None:
+                self.cur.close()
+        except Exception:
+            # traceback.print_exc()
+            pass
+
+        try:
+            if self.conn is not None:
+                self.conn.close()
+        except Exception:
+            # traceback.print_exc()
+            pass
+
+    def test_schedule_drop_job(self):
+        # schedule recurring job
+        self.cur.execute(
+                f'schedule job using notebook "{self.notebook_name}" '
+                'with mode "recurring" '
+                'execute every 5 minutes '
+                'with name "recurring-job" '
+                'create snapshot '
+                'resume target '
+                'with runtime "notebooks-cpu-small" '
+                'with parameters '
+                '{"strParam": "string", "intParam": 1, '
+                '"floatParam": 1.0, "boolParam": true}',
+        )
+        out = list(self.cur)
+        job_id = out[0][0]
+        self.job_ids.append(job_id)
+        desc = self.cur.description
+        assert len(desc) == 1
+        assert desc[0][0] == 'JobID'
+        assert len(out) == 1
+        assert out[0][0] == job_id
+
+        # drop job
+        self.cur.execute(f'drop jobs {job_id}')
+        out = list(self.cur)
+        desc = self.cur.description
+        assert len(desc) == 2
+        assert [x[0] for x in desc] == [
+            'JobID', 'Success',
+        ]
+        assert len(out) == 1
+        res = out[0]
+        assert res[0] == job_id
+        assert res[1] == 1
+
+    def test_run_wait_drop_job(self):
+        # run job
+        self.cur.execute(
+            f'run job using notebook "{self.notebook_name}" '
+            'with runtime "notebooks-cpu-small" '
+            'with parameters '
+            '{"strParam": "string", "intParam": 1, '
+            '"floatParam": 1.0, "boolParam": true}',
+        )
+        out = list(self.cur)
+        job_id = out[0][0]
+        self.job_ids.append(job_id)
+        desc = self.cur.description
+        assert len(desc) == 1
+        assert desc[0][0] == 'JobID'
+        assert len(out) == 1
+        assert out[0][0] == job_id
+
+        # wait on job
+        self.cur.execute(f'wait on jobs {job_id}')
+        out = list(self.cur)
+        desc = self.cur.description
+        assert len(desc) == 1
+        assert desc[0][0] == 'Success'
+        assert out[0][0] == 1
+
+        # drop job
+        self.cur.execute(f'drop jobs {job_id}')
+        out = list(self.cur)
+        desc = self.cur.description
+        assert len(desc) == 2
+        assert [x[0] for x in desc] == [
+            'JobID', 'Success',
+        ]
+        assert len(out) == 1
+        res = out[0]
+        assert res[0] == job_id
+        assert res[1] == 1
+
+    def test_show_jobs_and_executions(self):
+        # schedule recurring job
+        self.cur.execute(
+                f'schedule job using notebook "{self.notebook_name}" '
+                'with mode "recurring" '
+                'execute every 5 minutes '
+                'with name "show-job" '
+                'with runtime "notebooks-cpu-small" '
+                'with parameters '
+                '{"strParam": "string", "intParam": 1, '
+                '"floatParam": 1.0, "boolParam": true}',
+        )
+        out = list(self.cur)
+        job_id = out[0][0]
+        self.job_ids.append(job_id)
+        desc = self.cur.description
+        assert len(desc) == 1
+        assert desc[0][0] == 'JobID'
+        assert len(out) == 1
+        assert out[0][0] == job_id
+
+        # show jobs with name like "show-job"
+        self.cur.execute(f'show jobs {job_id} like "show-job"')
+        out = list(self.cur)
+        desc = self.cur.description
+        assert len(desc) == 9
+        assert [x[0] for x in desc] == [
+            'JobID', 'Name', 'CreatedAt', 'EnqueuedBy',
+            'CompletedExecutions', 'NotebookPath', 'DatabaseName', 'TargetID',
+            'TargetType',
+        ]
+        assert len(out) == 1
+        job = out[0]
+        assert job[0] == job_id
+        assert job[1] == 'show-job'
+        assert job[5] == self.notebook_name
+        assert job[6] == self.dbname
+        assert job[7] == self.workspace.id
+        assert job[8] == 'Workspace'
+
+        # show jobs with name like "show-job" extended
+        self.cur.execute(f'show jobs {job_id} like "show-job" extended')
+        out = list(self.cur)
+        desc = self.cur.description
+        assert len(desc) == 17
+        assert [x[0] for x in desc] == [
+            'JobID', 'Name', 'CreatedAt', 'EnqueuedBy',
+            'CompletedExecutions', 'NotebookPath', 'DatabaseName', 'TargetID',
+            'TargetType', 'Description', 'TerminatedAt', 'CreateSnapshot',
+            'MaxDurationInMins', 'ExecutionIntervalInMins', 'Mode', 'StartAt',
+            'ResumeTarget',
+        ]
+        assert len(out) == 1
+        job = out[0]
+        assert job[0] == job_id
+        assert job[1] == 'show-job'
+        assert job[5] == self.notebook_name
+        assert job[6] == self.dbname
+        assert job[7] == self.workspace.id
+        assert job[8] == 'Workspace'
+        assert not job[11]
+        assert job[13] == 5
+        assert job[14] == 'Recurring'
+        assert not job[16]
+
+        # show executions for job with id job_id from 1 to 5
+        self.cur.execute(f'show job executions for {job_id} from 1 to 5')
+        out = list(self.cur)
+        desc = self.cur.description
+        assert len(desc) == 7
+        assert [x[0] for x in desc] == [
+            'ExecutionID', 'ExecutionNumber', 'JobID',
+            'Status', 'ScheduledStartTime', 'StartedAt', 'FinishedAt',
+        ]
+        exec_job_ids = [x[2] for x in out]
+        assert [x for x in exec_job_ids] == [job_id]
+
+        # show executions for job with id job_id from 1 to 5 extended
+        self.cur.execute(f'show job executions for {job_id} from 1 to 5 extended')
+        out = list(self.cur)
+        desc = self.cur.description
+        assert len(desc) == 8
+        assert [x[0] for x in desc] == [
+            'ExecutionID', 'ExecutionNumber', 'JobID',
+            'Status', 'ScheduledStartTime', 'StartedAt', 'FinishedAt',
+            'SnapshotNotebookPath',
+        ]
+        exec_job_ids = [x[2] for x in out]
+        assert [x for x in exec_job_ids] == [job_id]
+
+        # drop job
+        self.cur.execute(f'drop jobs {job_id}')
+        out = list(self.cur)
+        desc = self.cur.description
+        assert len(desc) == 2
+        assert [x[0] for x in desc] == [
+            'JobID', 'Success',
+        ]
+        assert len(out) == 1
+        res = out[0]
+        assert res[0] == job_id
+        assert res[1] == 1

singlestoredb/tests/test_management.py

@@ -11,6 +11,8 @@ import unittest
 import pytest
 
 import singlestoredb as s2
+from singlestoredb.management.job import Status
+from singlestoredb.management.job import TargetType
 
 
 TEST_DIR = pathlib.Path(os.path.dirname(__file__))
@@ -876,3 +878,153 @@ class TestSecrets(unittest.TestCase):
 
         assert secret.name == 'secret_name'
         assert secret.value == 'secret_value'
+
+
+@pytest.mark.management
+class TestJob(unittest.TestCase):
+
+    manager = None
+    workspace_group = None
+    workspace = None
+    password = None
+    job_ids = []
+
+    @classmethod
+    def setUpClass(cls):
+        cls.manager = s2.manage_workspaces()
+
+        us_regions = [x for x in cls.manager.regions if 'US' in x.name]
+        cls.password = secrets.token_urlsafe(20)
+
+        name = clean_name(secrets.token_urlsafe(20)[:20])
+
+        cls.workspace_group = cls.manager.create_workspace_group(
+            f'wg-test-{name}',
+            region=random.choice(us_regions).id,
+            admin_password=cls.password,
+            firewall_ranges=['0.0.0.0/0'],
+        )
+
+        try:
+            cls.workspace = cls.workspace_group.create_workspace(
+                f'ws-test-{name}-x',
+                wait_on_active=True,
+            )
+        except Exception:
+            cls.workspace_group.terminate(force=True)
+            raise
+
+    @classmethod
+    def tearDownClass(cls):
+        for job_id in cls.job_ids:
+            cls.manager.organizations.current.jobs.delete(job_id)
+        if cls.workspace_group is not None:
+            cls.workspace_group.terminate(force=True)
+        cls.workspace_group = None
+        cls.workspace = None
+        cls.manager = None
+        cls.password = None
+        if os.environ.get('SINGLESTOREDB_WORKSPACE', None) is not None:
+            del os.environ['SINGLESTOREDB_WORKSPACE']
+        if os.environ.get('SINGLESTOREDB_DEFAULT_DATABASE', None) is not None:
+            del os.environ['SINGLESTOREDB_DEFAULT_DATABASE']
+
+    def test_job_without_database_target(self):
+        """
+        Creates job without target database on a specific runtime
+        Waits for job to finish
+        Gets the job
+        Deletes the job
+        """
+        if os.environ.get('SINGLESTOREDB_WORKSPACE', None) is not None:
+            del os.environ['SINGLESTOREDB_WORKSPACE']
+        if os.environ.get('SINGLESTOREDB_DEFAULT_DATABASE', None) is not None:
+            del os.environ['SINGLESTOREDB_DEFAULT_DATABASE']
+
+        job_manager = self.manager.organizations.current.jobs
+        job = job_manager.run(
+            'Scheduling Test.ipynb',
+            'notebooks-cpu-small',
+            {'strParam': 'string', 'intParam': 1, 'floatParam': 1.0, 'boolParam': True},
+        )
+        self.job_ids.append(job.job_id)
+        assert job.execution_config.notebook_path == 'Scheduling Test.ipynb'
+        assert job.schedule.mode == job_manager.modes().ONCE
+        assert not job.execution_config.create_snapshot
+        assert job.completed_executions_count == 0
+        assert job.name is None
+        assert job.description is None
+        assert job.job_metadata == []
+        assert job.terminated_at is None
+        assert job.target_config is None
+        job.wait()
+        job = job_manager.get(job.job_id)
+        assert job.execution_config.notebook_path == 'Scheduling Test.ipynb'
+        assert job.schedule.mode == job_manager.modes().ONCE
+        assert not job.execution_config.create_snapshot
+        assert job.completed_executions_count == 1
+        assert job.name is None
+        assert job.description is None
+        assert job.job_metadata != []
+        assert len(job.job_metadata) == 1
+        assert job.job_metadata[0].count == 1
+        assert job.job_metadata[0].status == Status.COMPLETED
+        assert job.terminated_at is None
+        assert job.target_config is None
+        deleted = job.delete()
+        assert deleted
+        job = job_manager.get(job.job_id)
+        assert job.terminated_at is not None
+
+    def test_job_with_database_target(self):
+        """
+        Creates job with target database on a specific runtime
+        Waits for job to finish
+        Gets the job
+        Deletes the job
+        """
+        os.environ['SINGLESTOREDB_DEFAULT_DATABASE'] = 'information_schema'
+        os.environ['SINGLESTOREDB_WORKSPACE'] = self.workspace.id
+
+        job_manager = self.manager.organizations.current.jobs
+        job = job_manager.run(
+            'Scheduling Test.ipynb',
+            'notebooks-cpu-small',
+            {'strParam': 'string', 'intParam': 1, 'floatParam': 1.0, 'boolParam': True},
+        )
+        self.job_ids.append(job.job_id)
+        assert job.execution_config.notebook_path == 'Scheduling Test.ipynb'
+        assert job.schedule.mode == job_manager.modes().ONCE
+        assert not job.execution_config.create_snapshot
+        assert job.completed_executions_count == 0
+        assert job.name is None
+        assert job.description is None
+        assert job.job_metadata == []
+        assert job.terminated_at is None
+        assert job.target_config is not None
+        assert job.target_config.database_name == 'information_schema'
+        assert job.target_config.target_id == self.workspace.id
+        assert job.target_config.target_type == TargetType.WORKSPACE
+        assert not job.target_config.resume_target
+        job.wait()
+        job = job_manager.get(job.job_id)
+        assert job.execution_config.notebook_path == 'Scheduling Test.ipynb'
+        assert job.schedule.mode == job_manager.modes().ONCE
+        assert not job.execution_config.create_snapshot
+        assert job.completed_executions_count == 1
+        assert job.name is None
+        assert job.description is None
+        assert job.job_metadata != []
+        assert len(job.job_metadata) == 1
+        assert job.job_metadata[0].count == 1
+        assert job.job_metadata[0].status == Status.COMPLETED
+        assert job.terminated_at is None
+        assert job.target_config is not None
+        assert job.target_config.database_name == 'information_schema'
+        assert job.target_config.target_id == self.workspace.id
+        assert job.target_config.target_type == TargetType.WORKSPACE
+        assert not job.target_config.resume_target
+        deleted = job.delete()
+        assert deleted
+        job = job_manager.get(job.job_id)
+        assert job.terminated_at is not None

singlestoredb/tests/utils.py

@@ -73,6 +73,13 @@ def load_sql(sql_file: str) -> str:
     with open(sql_file, 'r') as infile:
         with s2.connect(**args) as conn:
             with conn.cursor() as cur:
+                try:
+                    cur.execute('SET GLOBAL default_partitions_per_leaf=2')
+                    cur.execute('SET GLOBAL log_file_size_partitions=1048576')
+                    cur.execute('SET GLOBAL log_file_size_ref_dbs=1048576')
+                except s2.OperationalError:
+                    pass
+
                 if not dbname:
                     dbname = 'TEST_{}'.format(uuid.uuid4()).replace('-', '_')
                     cur.execute(f'CREATE DATABASE {dbname};')

{singlestoredb-1.4.3.dist-info → singlestoredb-1.6.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: singlestoredb
-Version: 1.4.3
+Version: 1.6.0
 Summary: Interface to the SingleStoreDB database and workspace management APIs
 Home-page: https://github.com/singlestore-labs/singlestoredb-python
 Author: SingleStore