singlestoredb 0.4.0__py3-none-any.whl → 1.0.4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of singlestoredb might be problematic. Click here for more details.

Files changed (120) hide show
  1. singlestoredb/__init__.py +33 -1
  2. singlestoredb/alchemy/__init__.py +90 -0
  3. singlestoredb/auth.py +5 -1
  4. singlestoredb/config.py +116 -14
  5. singlestoredb/connection.py +483 -516
  6. singlestoredb/converters.py +238 -135
  7. singlestoredb/exceptions.py +30 -2
  8. singlestoredb/functions/__init__.py +1 -0
  9. singlestoredb/functions/decorator.py +142 -0
  10. singlestoredb/functions/dtypes.py +1639 -0
  11. singlestoredb/functions/ext/__init__.py +2 -0
  12. singlestoredb/functions/ext/arrow.py +375 -0
  13. singlestoredb/functions/ext/asgi.py +661 -0
  14. singlestoredb/functions/ext/json.py +427 -0
  15. singlestoredb/functions/ext/mmap.py +306 -0
  16. singlestoredb/functions/ext/rowdat_1.py +744 -0
  17. singlestoredb/functions/signature.py +673 -0
  18. singlestoredb/fusion/__init__.py +11 -0
  19. singlestoredb/fusion/graphql.py +213 -0
  20. singlestoredb/fusion/handler.py +621 -0
  21. singlestoredb/fusion/handlers/stage.py +257 -0
  22. singlestoredb/fusion/handlers/utils.py +162 -0
  23. singlestoredb/fusion/handlers/workspace.py +412 -0
  24. singlestoredb/fusion/registry.py +164 -0
  25. singlestoredb/fusion/result.py +399 -0
  26. singlestoredb/http/__init__.py +27 -0
  27. singlestoredb/{http.py → http/connection.py} +555 -154
  28. singlestoredb/management/__init__.py +3 -0
  29. singlestoredb/management/billing_usage.py +148 -0
  30. singlestoredb/management/cluster.py +14 -6
  31. singlestoredb/management/manager.py +100 -38
  32. singlestoredb/management/organization.py +188 -0
  33. singlestoredb/management/region.py +5 -5
  34. singlestoredb/management/utils.py +281 -2
  35. singlestoredb/management/workspace.py +1344 -49
  36. singlestoredb/{clients/pymysqlsv → mysql}/__init__.py +16 -21
  37. singlestoredb/{clients/pymysqlsv → mysql}/_auth.py +39 -8
  38. singlestoredb/{clients/pymysqlsv → mysql}/charset.py +26 -23
  39. singlestoredb/{clients/pymysqlsv/connections.py → mysql/connection.py} +532 -165
  40. singlestoredb/{clients/pymysqlsv → mysql}/constants/CLIENT.py +0 -1
  41. singlestoredb/{clients/pymysqlsv → mysql}/constants/COMMAND.py +0 -1
  42. singlestoredb/{clients/pymysqlsv → mysql}/constants/CR.py +0 -2
  43. singlestoredb/{clients/pymysqlsv → mysql}/constants/ER.py +0 -1
  44. singlestoredb/{clients/pymysqlsv → mysql}/constants/FIELD_TYPE.py +1 -1
  45. singlestoredb/{clients/pymysqlsv → mysql}/constants/FLAG.py +0 -1
  46. singlestoredb/{clients/pymysqlsv → mysql}/constants/SERVER_STATUS.py +0 -1
  47. singlestoredb/mysql/converters.py +271 -0
  48. singlestoredb/{clients/pymysqlsv → mysql}/cursors.py +228 -112
  49. singlestoredb/mysql/err.py +92 -0
  50. singlestoredb/{clients/pymysqlsv → mysql}/optionfile.py +5 -4
  51. singlestoredb/{clients/pymysqlsv → mysql}/protocol.py +49 -20
  52. singlestoredb/mysql/tests/__init__.py +19 -0
  53. singlestoredb/{clients/pymysqlsv → mysql}/tests/base.py +32 -12
  54. singlestoredb/mysql/tests/conftest.py +37 -0
  55. singlestoredb/{clients/pymysqlsv → mysql}/tests/test_DictCursor.py +11 -7
  56. singlestoredb/{clients/pymysqlsv → mysql}/tests/test_SSCursor.py +17 -12
  57. singlestoredb/{clients/pymysqlsv → mysql}/tests/test_basic.py +32 -24
  58. singlestoredb/{clients/pymysqlsv → mysql}/tests/test_connection.py +130 -119
  59. singlestoredb/{clients/pymysqlsv → mysql}/tests/test_converters.py +9 -7
  60. singlestoredb/mysql/tests/test_cursor.py +141 -0
  61. singlestoredb/{clients/pymysqlsv → mysql}/tests/test_err.py +3 -2
  62. singlestoredb/{clients/pymysqlsv → mysql}/tests/test_issues.py +35 -27
  63. singlestoredb/{clients/pymysqlsv → mysql}/tests/test_load_local.py +13 -11
  64. singlestoredb/{clients/pymysqlsv → mysql}/tests/test_nextset.py +7 -3
  65. singlestoredb/{clients/pymysqlsv → mysql}/tests/test_optionfile.py +2 -1
  66. singlestoredb/{clients/pymysqlsv → mysql}/tests/thirdparty/__init__.py +1 -1
  67. singlestoredb/mysql/tests/thirdparty/test_MySQLdb/__init__.py +9 -0
  68. singlestoredb/{clients/pymysqlsv → mysql}/tests/thirdparty/test_MySQLdb/capabilities.py +19 -17
  69. singlestoredb/{clients/pymysqlsv → mysql}/tests/thirdparty/test_MySQLdb/dbapi20.py +31 -22
  70. singlestoredb/{clients/pymysqlsv → mysql}/tests/thirdparty/test_MySQLdb/test_MySQLdb_capabilities.py +3 -4
  71. singlestoredb/{clients/pymysqlsv → mysql}/tests/thirdparty/test_MySQLdb/test_MySQLdb_dbapi20.py +24 -20
  72. singlestoredb/{clients/pymysqlsv → mysql}/tests/thirdparty/test_MySQLdb/test_MySQLdb_nonstandard.py +4 -4
  73. singlestoredb/{clients/pymysqlsv → mysql}/times.py +3 -4
  74. singlestoredb/pytest.py +283 -0
  75. singlestoredb/tests/empty.sql +0 -0
  76. singlestoredb/tests/ext_funcs/__init__.py +385 -0
  77. singlestoredb/tests/test.sql +210 -0
  78. singlestoredb/tests/test2.sql +1 -0
  79. singlestoredb/tests/test_basics.py +482 -115
  80. singlestoredb/tests/test_config.py +13 -13
  81. singlestoredb/tests/test_connection.py +241 -305
  82. singlestoredb/tests/test_dbapi.py +27 -0
  83. singlestoredb/tests/test_ext_func.py +1193 -0
  84. singlestoredb/tests/test_ext_func_data.py +1101 -0
  85. singlestoredb/tests/test_fusion.py +465 -0
  86. singlestoredb/tests/test_http.py +32 -26
  87. singlestoredb/tests/test_management.py +588 -8
  88. singlestoredb/tests/test_plugin.py +33 -0
  89. singlestoredb/tests/test_results.py +11 -12
  90. singlestoredb/tests/test_udf.py +687 -0
  91. singlestoredb/tests/utils.py +3 -2
  92. singlestoredb/utils/config.py +58 -0
  93. singlestoredb/utils/debug.py +13 -0
  94. singlestoredb/utils/mogrify.py +151 -0
  95. singlestoredb/utils/results.py +4 -1
  96. singlestoredb-1.0.4.dist-info/METADATA +139 -0
  97. singlestoredb-1.0.4.dist-info/RECORD +112 -0
  98. {singlestoredb-0.4.0.dist-info → singlestoredb-1.0.4.dist-info}/WHEEL +1 -1
  99. singlestoredb-1.0.4.dist-info/entry_points.txt +2 -0
  100. singlestoredb/clients/pymysqlsv/converters.py +0 -365
  101. singlestoredb/clients/pymysqlsv/err.py +0 -144
  102. singlestoredb/clients/pymysqlsv/tests/__init__.py +0 -19
  103. singlestoredb/clients/pymysqlsv/tests/test_cursor.py +0 -133
  104. singlestoredb/clients/pymysqlsv/tests/thirdparty/test_MySQLdb/__init__.py +0 -9
  105. singlestoredb/drivers/__init__.py +0 -45
  106. singlestoredb/drivers/base.py +0 -198
  107. singlestoredb/drivers/cymysql.py +0 -38
  108. singlestoredb/drivers/http.py +0 -47
  109. singlestoredb/drivers/mariadb.py +0 -40
  110. singlestoredb/drivers/mysqlconnector.py +0 -49
  111. singlestoredb/drivers/mysqldb.py +0 -60
  112. singlestoredb/drivers/pymysql.py +0 -37
  113. singlestoredb/drivers/pymysqlsv.py +0 -35
  114. singlestoredb/drivers/pyodbc.py +0 -65
  115. singlestoredb-0.4.0.dist-info/METADATA +0 -111
  116. singlestoredb-0.4.0.dist-info/RECORD +0 -86
  117. /singlestoredb/{clients → fusion/handlers}/__init__.py +0 -0
  118. /singlestoredb/{clients/pymysqlsv → mysql}/constants/__init__.py +0 -0
  119. {singlestoredb-0.4.0.dist-info → singlestoredb-1.0.4.dist-info}/LICENSE +0 -0
  120. {singlestoredb-0.4.0.dist-info → singlestoredb-1.0.4.dist-info}/top_level.txt +0 -0
@@ -1,20 +1,913 @@
1
1
  #!/usr/bin/env python
2
2
  """SingleStoreDB Workspace Management."""
3
+ from __future__ import annotations
4
+
3
5
  import datetime
6
+ import glob
7
+ import io
8
+ import os
9
+ import re
10
+ import time
11
+ from collections.abc import Mapping
4
12
  from typing import Any
13
+ from typing import BinaryIO
5
14
  from typing import Dict
6
15
  from typing import List
7
16
  from typing import Optional
17
+ from typing import TextIO
8
18
  from typing import Union
9
19
 
10
20
  from .. import connection
11
21
  from ..exceptions import ManagementError
22
+ from .billing_usage import BillingUsageItem
12
23
  from .manager import Manager
24
+ from .organization import Organization
13
25
  from .region import Region
26
+ from .utils import camel_to_snake_dict
27
+ from .utils import from_datetime
28
+ from .utils import NamedList
29
+ from .utils import PathLike
30
+ from .utils import snake_to_camel
31
+ from .utils import snake_to_camel_dict
14
32
  from .utils import to_datetime
33
+ from .utils import ttl_property
15
34
  from .utils import vars_to_str
16
35
 
17
36
 
37
+ def get_secret(name: str) -> str:
38
+ """Get a secret from the organization."""
39
+ return manage_workspaces().organization.get_secret(name).value
40
+
41
+
42
+ class StageObject(object):
43
+ """
44
+ Stage file / folder object.
45
+
46
+ This object is not instantiated directly. It is used in the results
47
+ of various operations in ``WorkspaceGroup.stage`` methods.
48
+
49
+ """
50
+
51
+ def __init__(
52
+ self,
53
+ name: str,
54
+ path: str,
55
+ size: int,
56
+ type: str,
57
+ format: str,
58
+ mimetype: str,
59
+ created: Optional[datetime.datetime],
60
+ last_modified: Optional[datetime.datetime],
61
+ writable: bool,
62
+ content: Optional[List[str]] = None,
63
+ ):
64
+ #: Name of file / folder
65
+ self.name = name
66
+
67
+ if type == 'directory':
68
+ path = re.sub(r'/*$', r'', str(path)) + '/'
69
+
70
+ #: Path of file / folder
71
+ self.path = path
72
+
73
+ #: Size of the object (in bytes)
74
+ self.size = size
75
+
76
+ #: Data type: file or directory
77
+ self.type = type
78
+
79
+ #: Data format
80
+ self.format = format
81
+
82
+ #: Mime type
83
+ self.mimetype = mimetype
84
+
85
+ #: Datetime the object was created
86
+ self.created_at = created
87
+
88
+ #: Datetime the object was modified last
89
+ self.last_modified_at = last_modified
90
+
91
+ #: Is the object writable?
92
+ self.writable = writable
93
+
94
+ #: Contents of a directory
95
+ self.content: List[str] = content or []
96
+
97
+ self._stage: Optional[Stage] = None
98
+
99
+ @classmethod
100
+ def from_dict(
101
+ cls,
102
+ obj: Dict[str, Any],
103
+ stage: Stage,
104
+ ) -> StageObject:
105
+ """
106
+ Construct a StageObject from a dictionary of values.
107
+
108
+ Parameters
109
+ ----------
110
+ obj : dict
111
+ Dictionary of values
112
+ stage : Stage
113
+ Stage object to use as the parent
114
+
115
+ Returns
116
+ -------
117
+ :class:`StageObject`
118
+
119
+ """
120
+ out = cls(
121
+ name=obj['name'],
122
+ path=obj['path'],
123
+ size=obj['size'],
124
+ type=obj['type'],
125
+ format=obj['format'],
126
+ mimetype=obj['mimetype'],
127
+ created=to_datetime(obj.get('created')),
128
+ last_modified=to_datetime(obj.get('last_modified')),
129
+ writable=bool(obj['writable']),
130
+ )
131
+ out._stage = stage
132
+ return out
133
+
134
+ def __str__(self) -> str:
135
+ """Return string representation."""
136
+ return vars_to_str(self)
137
+
138
+ def __repr__(self) -> str:
139
+ """Return string representation."""
140
+ return str(self)
141
+
142
+ def open(
143
+ self,
144
+ mode: str = 'r',
145
+ encoding: Optional[str] = None,
146
+ ) -> Union[io.StringIO, io.BytesIO]:
147
+ """
148
+ Open a Stage path for reading or writing.
149
+
150
+ Parameters
151
+ ----------
152
+ mode : str, optional
153
+ The read / write mode. The following modes are supported:
154
+ * 'r' open for reading (default)
155
+ * 'w' open for writing, truncating the file first
156
+ * 'x' create a new file and open it for writing
157
+ The data type can be specified by adding one of the following:
158
+ * 'b' binary mode
159
+ * 't' text mode (default)
160
+ encoding : str, optional
161
+ The string encoding to use for text
162
+
163
+ Returns
164
+ -------
165
+ StageObjectBytesReader - 'rb' or 'b' mode
166
+ StageObjectBytesWriter - 'wb' or 'xb' mode
167
+ StageObjectTextReader - 'r' or 'rt' mode
168
+ StageObjectTextWriter - 'w', 'x', 'wt' or 'xt' mode
169
+
170
+ """
171
+ if self._stage is None:
172
+ raise ManagementError(
173
+ msg='No Stage object is associated with this object.',
174
+ )
175
+
176
+ if self.is_dir():
177
+ raise IsADirectoryError(
178
+ f'directories can not be read or written: {self.path}',
179
+ )
180
+
181
+ return self._stage.open(self.path, mode=mode, encoding=encoding)
182
+
183
+ def download(
184
+ self,
185
+ local_path: Optional[PathLike] = None,
186
+ *,
187
+ overwrite: bool = False,
188
+ encoding: Optional[str] = None,
189
+ ) -> Optional[Union[bytes, str]]:
190
+ """
191
+ Download the content of a stage path.
192
+
193
+ Parameters
194
+ ----------
195
+ local_path : Path or str
196
+ Path to local file target location
197
+ overwrite : bool, optional
198
+ Should an existing file be overwritten if it exists?
199
+ encoding : str, optional
200
+ Encoding used to convert the resulting data
201
+
202
+ Returns
203
+ -------
204
+ bytes or str or None
205
+
206
+ """
207
+ if self._stage is None:
208
+ raise ManagementError(
209
+ msg='No Stage object is associated with this object.',
210
+ )
211
+
212
+ return self._stage.download_file(
213
+ self.path, local_path=local_path,
214
+ overwrite=overwrite, encoding=encoding,
215
+ )
216
+
217
+ download_file = download
218
+
219
+ def remove(self) -> None:
220
+ """Delete the stage file."""
221
+ if self._stage is None:
222
+ raise ManagementError(
223
+ msg='No Stage object is associated with this object.',
224
+ )
225
+
226
+ if self.type == 'directory':
227
+ raise IsADirectoryError(
228
+ f'path is a directory; use rmdir or removedirs {self.path}',
229
+ )
230
+
231
+ self._stage.remove(self.path)
232
+
233
+ def rmdir(self) -> None:
234
+ """Delete the empty stage directory."""
235
+ if self._stage is None:
236
+ raise ManagementError(
237
+ msg='No Stage object is associated with this object.',
238
+ )
239
+
240
+ if self.type != 'directory':
241
+ raise NotADirectoryError(
242
+ f'path is not a directory: {self.path}',
243
+ )
244
+
245
+ self._stage.rmdir(self.path)
246
+
247
+ def removedirs(self) -> None:
248
+ """Delete the stage directory recursively."""
249
+ if self._stage is None:
250
+ raise ManagementError(
251
+ msg='No Stage object is associated with this object.',
252
+ )
253
+
254
+ if self.type != 'directory':
255
+ raise NotADirectoryError(
256
+ f'path is not a directory: {self.path}',
257
+ )
258
+
259
+ self._stage.removedirs(self.path)
260
+
261
+ def rename(self, new_path: PathLike, *, overwrite: bool = False) -> None:
262
+ """
263
+ Move the stage file to a new location.
264
+
265
+ Parameters
266
+ ----------
267
+ new_path : Path or str
268
+ The new location of the file
269
+ overwrite : bool, optional
270
+ Should path be overwritten if it already exists?
271
+
272
+ """
273
+ if self._stage is None:
274
+ raise ManagementError(
275
+ msg='No Stage object is associated with this object.',
276
+ )
277
+ out = self._stage.rename(self.path, new_path, overwrite=overwrite)
278
+ self.name = out.name
279
+ self.path = out.path
280
+ return None
281
+
282
+ def exists(self) -> bool:
283
+ """Does the file / folder exist?"""
284
+ if self._stage is None:
285
+ raise ManagementError(
286
+ msg='No Stage object is associated with this object.',
287
+ )
288
+ return self._stage.exists(self.path)
289
+
290
+ def is_dir(self) -> bool:
291
+ """Is the stage object a directory?"""
292
+ return self.type == 'directory'
293
+
294
+ def is_file(self) -> bool:
295
+ """Is the stage object a file?"""
296
+ return self.type != 'directory'
297
+
298
+ def abspath(self) -> str:
299
+ """Return the full path of the object."""
300
+ return str(self.path)
301
+
302
+ def basename(self) -> str:
303
+ """Return the basename of the object."""
304
+ return self.name
305
+
306
+ def dirname(self) -> str:
307
+ """Return the directory name of the object."""
308
+ return re.sub(r'/*$', r'', os.path.dirname(re.sub(r'/*$', r'', self.path))) + '/'
309
+
310
+ def getmtime(self) -> float:
311
+ """Return the last modified datetime as a UNIX timestamp."""
312
+ if self.last_modified_at is None:
313
+ return 0.0
314
+ return self.last_modified_at.timestamp()
315
+
316
+ def getctime(self) -> float:
317
+ """Return the creation datetime as a UNIX timestamp."""
318
+ if self.created_at is None:
319
+ return 0.0
320
+ return self.created_at.timestamp()
321
+
322
+
323
+ class StageObjectTextWriter(io.StringIO):
324
+ """StringIO wrapper for writing to Stage."""
325
+
326
+ def __init__(self, buffer: Optional[str], stage: Stage, stage_path: PathLike):
327
+ self._stage = stage
328
+ self._stage_path = stage_path
329
+ super().__init__(buffer)
330
+
331
+ def close(self) -> None:
332
+ """Write the content to the stage path."""
333
+ self._stage._upload(self.getvalue(), self._stage_path)
334
+ super().close()
335
+
336
+
337
+ class StageObjectTextReader(io.StringIO):
338
+ """StringIO wrapper for reading from Stage."""
339
+
340
+
341
+ class StageObjectBytesWriter(io.BytesIO):
342
+ """BytesIO wrapper for writing to Stage."""
343
+
344
+ def __init__(self, buffer: bytes, stage: Stage, stage_path: PathLike):
345
+ self._stage = stage
346
+ self._stage_path = stage_path
347
+ super().__init__(buffer)
348
+
349
+ def close(self) -> None:
350
+ """Write the content to the stage path."""
351
+ self._stage._upload(self.getvalue(), self._stage_path)
352
+ super().close()
353
+
354
+
355
+ class StageObjectBytesReader(io.BytesIO):
356
+ """BytesIO wrapper for reading from Stage."""
357
+
358
+
359
+ class Stage(object):
360
+ """
361
+ Stage manager.
362
+
363
+ This object is not instantiated directly.
364
+ It is returned by ``WorkspaceGroup.stage``.
365
+
366
+ """
367
+
368
+ def __init__(self, workspace_group: WorkspaceGroup, manager: WorkspaceManager):
369
+ self._workspace_group = workspace_group
370
+ self._manager = manager
371
+
372
+ def open(
373
+ self,
374
+ stage_path: PathLike,
375
+ mode: str = 'r',
376
+ encoding: Optional[str] = None,
377
+ ) -> Union[io.StringIO, io.BytesIO]:
378
+ """
379
+ Open a Stage path for reading or writing.
380
+
381
+ Parameters
382
+ ----------
383
+ stage_path : Path or str
384
+ The stage path to read / write
385
+ mode : str, optional
386
+ The read / write mode. The following modes are supported:
387
+ * 'r' open for reading (default)
388
+ * 'w' open for writing, truncating the file first
389
+ * 'x' create a new file and open it for writing
390
+ The data type can be specified by adding one of the following:
391
+ * 'b' binary mode
392
+ * 't' text mode (default)
393
+ encoding : str, optional
394
+ The string encoding to use for text
395
+
396
+ Returns
397
+ -------
398
+ StageObjectBytesReader - 'rb' or 'b' mode
399
+ StageObjectBytesWriter - 'wb' or 'xb' mode
400
+ StageObjectTextReader - 'r' or 'rt' mode
401
+ StageObjectTextWriter - 'w', 'x', 'wt' or 'xt' mode
402
+
403
+ """
404
+ if '+' in mode or 'a' in mode:
405
+ raise ValueError('modifying an existing stage file is not supported')
406
+
407
+ if 'w' in mode or 'x' in mode:
408
+ exists = self.exists(stage_path)
409
+ if exists:
410
+ if 'x' in mode:
411
+ raise FileExistsError(f'stage path already exists: {stage_path}')
412
+ self.remove(stage_path)
413
+ if 'b' in mode:
414
+ return StageObjectBytesWriter(b'', self, stage_path)
415
+ return StageObjectTextWriter('', self, stage_path)
416
+
417
+ if 'r' in mode:
418
+ content = self.download_file(stage_path)
419
+ if isinstance(content, bytes):
420
+ if 'b' in mode:
421
+ return StageObjectBytesReader(content)
422
+ encoding = 'utf-8' if encoding is None else encoding
423
+ return StageObjectTextReader(content.decode(encoding))
424
+
425
+ if isinstance(content, str):
426
+ return StageObjectTextReader(content)
427
+
428
+ raise ValueError(f'unrecognized file content type: {type(content)}')
429
+
430
+ raise ValueError(f'must have one of create/read/write mode specified: {mode}')
431
+
432
+ def upload_file(
433
+ self,
434
+ local_path: Union[PathLike, TextIO, BinaryIO],
435
+ stage_path: PathLike,
436
+ *,
437
+ overwrite: bool = False,
438
+ ) -> StageObject:
439
+ """
440
+ Upload a local file.
441
+
442
+ Parameters
443
+ ----------
444
+ local_path : Path or str or file-like
445
+ Path to the local file or an open file object
446
+ stage_path : Path or str
447
+ Path to the stage file
448
+ overwrite : bool, optional
449
+ Should the ``stage_path`` be overwritten if it exists already?
450
+
451
+ """
452
+ if isinstance(local_path, (TextIO, BinaryIO)):
453
+ pass
454
+ elif not os.path.isfile(local_path):
455
+ raise IsADirectoryError(f'local path is not a file: {local_path}')
456
+
457
+ if self.exists(stage_path):
458
+ if not overwrite:
459
+ raise OSError(f'stage path already exists: {stage_path}')
460
+
461
+ self.remove(stage_path)
462
+
463
+ if isinstance(local_path, (TextIO, BinaryIO)):
464
+ return self._upload(local_path, stage_path, overwrite=overwrite)
465
+ return self._upload(open(local_path, 'rb'), stage_path, overwrite=overwrite)
466
+
467
+ def upload_folder(
468
+ self,
469
+ local_path: PathLike,
470
+ stage_path: PathLike,
471
+ *,
472
+ overwrite: bool = False,
473
+ recursive: bool = True,
474
+ include_root: bool = False,
475
+ ignore: Optional[Union[PathLike, List[PathLike]]] = None,
476
+ ) -> StageObject:
477
+ """
478
+ Upload a folder recursively.
479
+
480
+ Only the contents of the folder are uploaded. To include the
481
+ folder name itself in the target path use ``include_root=True``.
482
+
483
+ Parameters
484
+ ----------
485
+ local_path : Path or str
486
+ Local directory to upload
487
+ stage_path : Path or str
488
+ Path of stage folder to upload to
489
+ overwrite : bool, optional
490
+ If a file already exists, should it be overwritten?
491
+ recursive : bool, optional
492
+ Should nested folders be uploaded?
493
+ include_root : bool, optional
494
+ Should the local root folder itself be uploaded as the top folder?
495
+ ignore : Path or str or List[Path] or List[str], optional
496
+ Glob patterns of files to ignore, for example, '**/*.pyc` will
497
+ ignore all '*.pyc' files in the directory tree
498
+
499
+ """
500
+ if not os.path.isdir(local_path):
501
+ raise NotADirectoryError(f'local path is not a directory: {local_path}')
502
+ if self.exists(stage_path) and not self.is_dir(stage_path):
503
+ raise NotADirectoryError(f'stage path is not a directory: {stage_path}')
504
+
505
+ ignore_files = set()
506
+ if ignore:
507
+ if isinstance(ignore, list):
508
+ for item in ignore:
509
+ ignore_files.update(glob.glob(str(item), recursive=recursive))
510
+ else:
511
+ ignore_files.update(glob.glob(str(ignore), recursive=recursive))
512
+
513
+ parent_dir = os.path.basename(os.getcwd())
514
+
515
+ files = glob.glob(os.path.join(local_path, '**'), recursive=recursive)
516
+
517
+ for src in files:
518
+ if ignore_files and src in ignore_files:
519
+ continue
520
+ target = os.path.join(parent_dir, src) if include_root else src
521
+ self.upload_file(src, target, overwrite=overwrite)
522
+
523
+ return self.info(stage_path)
524
+
525
+ def _upload(
526
+ self,
527
+ content: Union[str, bytes, TextIO, BinaryIO],
528
+ stage_path: PathLike,
529
+ *,
530
+ overwrite: bool = False,
531
+ ) -> StageObject:
532
+ """
533
+ Upload content to a stage file.
534
+
535
+ Parameters
536
+ ----------
537
+ content : str or bytes or file-like
538
+ Content to upload to stage
539
+ stage_path : Path or str
540
+ Path to the stage file
541
+ overwrite : bool, optional
542
+ Should the ``stage_path`` be overwritten if it exists already?
543
+
544
+ """
545
+ if self.exists(stage_path):
546
+ if not overwrite:
547
+ raise OSError(f'stage path already exists: {stage_path}')
548
+ self.remove(stage_path)
549
+
550
+ self._manager._put(
551
+ f'stage/{self._workspace_group.id}/fs/{stage_path}',
552
+ files={'file': content},
553
+ headers={'Content-Type': None},
554
+ )
555
+
556
+ return self.info(stage_path)
557
+
558
+ def mkdir(self, stage_path: PathLike, overwrite: bool = False) -> StageObject:
559
+ """
560
+ Make a directory in the stage.
561
+
562
+ Parameters
563
+ ----------
564
+ stage_path : Path or str
565
+ Path of the folder to create
566
+ overwrite : bool, optional
567
+ Should the stage path be overwritten if it exists already?
568
+
569
+ Returns
570
+ -------
571
+ StageObject
572
+
573
+ """
574
+ stage_path = re.sub(r'/*$', r'', str(stage_path)) + '/'
575
+
576
+ if self.exists(stage_path):
577
+ if not overwrite:
578
+ return self.info(stage_path)
579
+
580
+ self.remove(stage_path)
581
+
582
+ self._manager._put(
583
+ f'stage/{self._workspace_group.id}/fs/{stage_path}?isFile=false',
584
+ )
585
+
586
+ return self.info(stage_path)
587
+
588
+ mkdirs = mkdir
589
+
590
+ def rename(
591
+ self,
592
+ old_path: PathLike,
593
+ new_path: PathLike,
594
+ *,
595
+ overwrite: bool = False,
596
+ ) -> StageObject:
597
+ """
598
+ Move the stage file to a new location.
599
+
600
+ Paraemeters
601
+ -----------
602
+ old_path : Path or str
603
+ Original location of the path
604
+ new_path : Path or str
605
+ New location of the path
606
+ overwrite : bool, optional
607
+ Should the ``new_path`` be overwritten if it exists already?
608
+
609
+ """
610
+ if not self.exists(old_path):
611
+ raise OSError(f'stage path does not exist: {old_path}')
612
+
613
+ if self.exists(new_path):
614
+ if not overwrite:
615
+ raise OSError(f'stage path already exists: {new_path}')
616
+
617
+ if str(old_path).endswith('/') and not str(new_path).endswith('/'):
618
+ raise OSError('original and new paths are not the same type')
619
+
620
+ if str(new_path).endswith('/'):
621
+ self.removedirs(new_path)
622
+ else:
623
+ self.remove(new_path)
624
+
625
+ self._manager._patch(
626
+ f'stage/{self._workspace_group.id}/fs/{old_path}',
627
+ json=dict(newPath=new_path),
628
+ )
629
+
630
+ return self.info(new_path)
631
+
632
+ def info(self, stage_path: PathLike) -> StageObject:
633
+ """
634
+ Return information about a stage location.
635
+
636
+ Parameters
637
+ ----------
638
+ stage_path : Path or str
639
+ Path to the stage location
640
+
641
+ Returns
642
+ -------
643
+ StageObject
644
+
645
+ """
646
+ res = self._manager._get(
647
+ re.sub(r'/+$', r'/', f'stage/{self._workspace_group.id}/fs/{stage_path}'),
648
+ params=dict(metadata=1),
649
+ ).json()
650
+
651
+ return StageObject.from_dict(res, self)
652
+
653
+ def exists(self, stage_path: PathLike) -> bool:
654
+ """
655
+ Does the given stage path exist?
656
+
657
+ Parameters
658
+ ----------
659
+ stage_path : Path or str
660
+ Path to stage object
661
+
662
+ Returns
663
+ -------
664
+ bool
665
+
666
+ """
667
+ try:
668
+ self.info(stage_path)
669
+ return True
670
+ except ManagementError as exc:
671
+ if exc.errno == 404:
672
+ return False
673
+ raise
674
+
675
+ def is_dir(self, stage_path: PathLike) -> bool:
676
+ """
677
+ Is the given stage path a directory?
678
+
679
+ Parameters
680
+ ----------
681
+ stage_path : Path or str
682
+ Path to stage object
683
+
684
+ Returns
685
+ -------
686
+ bool
687
+
688
+ """
689
+ try:
690
+ return self.info(stage_path).type == 'directory'
691
+ except ManagementError as exc:
692
+ if exc.errno == 404:
693
+ return False
694
+ raise
695
+
696
+ def is_file(self, stage_path: PathLike) -> bool:
697
+ """
698
+ Is the given stage path a file?
699
+
700
+ Parameters
701
+ ----------
702
+ stage_path : Path or str
703
+ Path to stage object
704
+
705
+ Returns
706
+ -------
707
+ bool
708
+
709
+ """
710
+ try:
711
+ return self.info(stage_path).type != 'directory'
712
+ except ManagementError as exc:
713
+ if exc.errno == 404:
714
+ return False
715
+ raise
716
+
717
+ def _listdir(self, stage_path: PathLike, *, recursive: bool = False) -> List[str]:
718
+ """
719
+ Return the names of files in a directory.
720
+
721
+ Parameters
722
+ ----------
723
+ stage_path : Path or str
724
+ Path to the folder in Stage
725
+ recursive : bool, optional
726
+ Should folders be listed recursively?
727
+
728
+ """
729
+ res = self._manager._get(
730
+ f'stage/{self._workspace_group.id}/fs/{stage_path}',
731
+ ).json()
732
+ if recursive:
733
+ out = []
734
+ for item in res['content'] or []:
735
+ out.append(item['path'])
736
+ if item['type'] == 'directory':
737
+ out.extend(self._listdir(item['path'], recursive=recursive))
738
+ return out
739
+ return [x['path'] for x in res['content'] or []]
740
+
741
+ def listdir(
742
+ self,
743
+ stage_path: PathLike = '/',
744
+ *,
745
+ recursive: bool = False,
746
+ ) -> List[str]:
747
+ """
748
+ List the files / folders at the given path.
749
+
750
+ Parameters
751
+ ----------
752
+ stage_path : Path or str, optional
753
+ Path to the stage location
754
+
755
+ Returns
756
+ -------
757
+ List[str]
758
+
759
+ """
760
+ stage_path = re.sub(r'^(\./|/)+', r'', str(stage_path))
761
+ stage_path = re.sub(r'/+$', r'', stage_path) + '/'
762
+
763
+ if self.is_dir(stage_path):
764
+ out = self._listdir(stage_path, recursive=recursive)
765
+ if stage_path != '/':
766
+ stage_path_n = len(stage_path.split('/')) - 1
767
+ out = ['/'.join(x.split('/')[stage_path_n:]) for x in out]
768
+ return out
769
+
770
+ raise NotADirectoryError(f'stage path is not a directory: {stage_path}')
771
+
772
+ def download_file(
773
+ self,
774
+ stage_path: PathLike,
775
+ local_path: Optional[PathLike] = None,
776
+ *,
777
+ overwrite: bool = False,
778
+ encoding: Optional[str] = None,
779
+ ) -> Optional[Union[bytes, str]]:
780
+ """
781
+ Download the content of a stage path.
782
+
783
+ Parameters
784
+ ----------
785
+ stage_path : Path or str
786
+ Path to the stage file
787
+ local_path : Path or str
788
+ Path to local file target location
789
+ overwrite : bool, optional
790
+ Should an existing file be overwritten if it exists?
791
+ encoding : str, optional
792
+ Encoding used to convert the resulting data
793
+
794
+ Returns
795
+ -------
796
+ bytes or str - ``local_path`` is None
797
+ None - ``local_path`` is a Path or str
798
+
799
+ """
800
+ if local_path is not None and not overwrite and os.path.exists(local_path):
801
+ raise OSError('target file already exists; use overwrite=True to replace')
802
+ if self.is_dir(stage_path):
803
+ raise IsADirectoryError(f'stage path is a directory: {stage_path}')
804
+
805
+ out = self._manager._get(
806
+ f'stage/{self._workspace_group.id}/fs/{stage_path}',
807
+ ).content
808
+
809
+ if local_path is not None:
810
+ with open(local_path, 'wb') as outfile:
811
+ outfile.write(out)
812
+ return None
813
+
814
+ if encoding:
815
+ return out.decode(encoding)
816
+
817
+ return out
818
+
819
+ def download_folder(
820
+ self,
821
+ stage_path: PathLike,
822
+ local_path: PathLike = '.',
823
+ *,
824
+ overwrite: bool = False,
825
+ ) -> None:
826
+ """
827
+ Download a Stage folder to a local directory.
828
+
829
+ Parameters
830
+ ----------
831
+ stage_path : Path or str
832
+ Path to the stage file
833
+ local_path : Path or str
834
+ Path to local directory target location
835
+ overwrite : bool, optional
836
+ Should an existing directory / files be overwritten if they exist?
837
+
838
+ """
839
+ if local_path is not None and not overwrite and os.path.exists(local_path):
840
+ raise OSError(
841
+ 'target directory already exists; '
842
+ 'use overwrite=True to replace',
843
+ )
844
+ if not self.is_dir(stage_path):
845
+ raise NotADirectoryError(f'stage path is not a directory: {stage_path}')
846
+
847
+ for f in self.listdir(stage_path, recursive=True):
848
+ if self.is_dir(f):
849
+ continue
850
+ target = os.path.normpath(os.path.join(local_path, f))
851
+ os.makedirs(os.path.dirname(target), exist_ok=True)
852
+ self.download_file(f, target, overwrite=overwrite)
853
+
854
+ def remove(self, stage_path: PathLike) -> None:
855
+ """
856
+ Delete a stage location.
857
+
858
+ Parameters
859
+ ----------
860
+ stage_path : Path or str
861
+ Path to the stage location
862
+
863
+ """
864
+ if self.is_dir(stage_path):
865
+ raise IsADirectoryError(
866
+ 'stage path is a directory, '
867
+ f'use rmdir or removedirs: {stage_path}',
868
+ )
869
+
870
+ self._manager._delete(f'stage/{self._workspace_group.id}/fs/{stage_path}')
871
+
872
+ def removedirs(self, stage_path: PathLike) -> None:
873
+ """
874
+ Delete a stage folder recursively.
875
+
876
+ Parameters
877
+ ----------
878
+ stage_path : Path or str
879
+ Path to the stage location
880
+
881
+ """
882
+ stage_path = re.sub(r'/*$', r'', str(stage_path)) + '/'
883
+ self._manager._delete(f'stage/{self._workspace_group.id}/fs/{stage_path}')
884
+
885
+ def rmdir(self, stage_path: PathLike) -> None:
886
+ """
887
+ Delete a stage folder.
888
+
889
+ Parameters
890
+ ----------
891
+ stage_path : Path or str
892
+ Path to the stage location
893
+
894
+ """
895
+ stage_path = re.sub(r'/*$', r'', str(stage_path)) + '/'
896
+
897
+ if self.listdir(stage_path):
898
+ raise OSError(f'stage folder is not empty, use removedirs: {stage_path}')
899
+
900
+ self._manager._delete(f'stage/{self._workspace_group.id}/fs/{stage_path}')
901
+
902
+ def __str__(self) -> str:
903
+ """Return string representation."""
904
+ return vars_to_str(self)
905
+
906
+ def __repr__(self) -> str:
907
+ """Return string representation."""
908
+ return str(self)
909
+
910
+
18
911
  class Workspace(object):
19
912
  """
20
913
  SingleStoreDB workspace definition.
@@ -34,12 +927,21 @@ class Workspace(object):
34
927
  """
35
928
 
36
929
  def __init__(
37
- self, name: str, workspace_id: str,
930
+ self,
931
+ name: str,
932
+ workspace_id: str,
38
933
  workspace_group: Union[str, 'WorkspaceGroup'],
39
- size: str, state: str,
934
+ size: str,
935
+ state: str,
40
936
  created_at: Union[str, datetime.datetime],
41
937
  terminated_at: Optional[Union[str, datetime.datetime]] = None,
42
938
  endpoint: Optional[str] = None,
939
+ auto_suspend: Optional[Dict[str, Any]] = None,
940
+ cache_config: Optional[int] = None,
941
+ deployment_type: Optional[str] = None,
942
+ resume_attachments: Optional[Dict[str, Any]] = None,
943
+ scaling_progress: Optional[int] = None,
944
+ last_resumed_at: Optional[str] = None,
43
945
  ):
44
946
  #: Name of the workspace
45
947
  self.name = name
@@ -69,6 +971,24 @@ class Workspace(object):
69
971
  #: Hostname (or IP address) of the workspace database server
70
972
  self.endpoint = endpoint
71
973
 
974
+ #: Current auto-suspend settings
975
+ self.auto_suspend = camel_to_snake_dict(auto_suspend)
976
+
977
+ #: Multiplier for the persistent cache
978
+ self.cache_config = cache_config
979
+
980
+ #: Deployment type of the workspace
981
+ self.deployment_type = deployment_type
982
+
983
+ #: Database attachments
984
+ self.resume_attachments = camel_to_snake_dict(resume_attachments)
985
+
986
+ #: Current progress percentage for scaling the workspace
987
+ self.scaling_progress = scaling_progress
988
+
989
+ #: Timestamp when workspace was last resumed
990
+ self.last_resumed_at = to_datetime(last_resumed_at)
991
+
72
992
  self._manager: Optional[WorkspaceManager] = None
73
993
 
74
994
  def __str__(self) -> str:
@@ -97,16 +1017,65 @@ class Workspace(object):
97
1017
 
98
1018
  """
99
1019
  out = cls(
100
- name=obj['name'], workspace_id=obj['workspaceID'],
1020
+ name=obj['name'],
1021
+ workspace_id=obj['workspaceID'],
101
1022
  workspace_group=obj['workspaceGroupID'],
102
- size=obj.get('size', 'Unknown'), state=obj['state'],
103
- created_at=obj['createdAt'], terminated_at=obj.get('terminatedAt'),
1023
+ size=obj.get('size', 'Unknown'),
1024
+ state=obj['state'],
1025
+ created_at=obj['createdAt'],
1026
+ terminated_at=obj.get('terminatedAt'),
104
1027
  endpoint=obj.get('endpoint'),
1028
+ auto_suspend=obj.get('autoSuspend'),
1029
+ cache_config=obj.get('cacheConfig'),
1030
+ deployment_type=obj.get('deploymentType'),
1031
+ last_resumed_at=obj.get('lastResumedAt'),
1032
+ resume_attachments=obj.get('resumeAttachments'),
1033
+ scaling_progress=obj.get('scalingProgress'),
105
1034
  )
106
1035
  out._manager = manager
107
1036
  return out
108
1037
 
109
- def refresh(self) -> 'Workspace':
1038
+ def update(
1039
+ self,
1040
+ auto_suspend: Optional[Dict[str, Any]] = None,
1041
+ cache_config: Optional[int] = None,
1042
+ deployment_type: Optional[str] = None,
1043
+ size: Optional[str] = None,
1044
+ ) -> None:
1045
+ """
1046
+ Update the workspace definition.
1047
+
1048
+ Parameters
1049
+ ----------
1050
+ auto_suspend : Dict[str, Any], optional
1051
+ Auto-suspend mode for the workspace: IDLE, SCHEDULED, DISABLED
1052
+ cache_config : int, optional
1053
+ Specifies the multiplier for the persistent cache associated
1054
+ with the workspace. If specified, it enables the cache configuration
1055
+ multiplier. It can have one of the following values: 1, 2, or 4.
1056
+ deployment_type : str, optional
1057
+ The deployment type that will be applied to all the workspaces
1058
+ within the group
1059
+ size : str, optional
1060
+ Size of the workspace (in workspace size notation), such as "S-1".
1061
+
1062
+ """
1063
+ if self._manager is None:
1064
+ raise ManagementError(
1065
+ msg='No workspace manager is associated with this object.',
1066
+ )
1067
+ data = {
1068
+ k: v for k, v in dict(
1069
+ autoSuspend=snake_to_camel_dict(auto_suspend),
1070
+ cacheConfig=cache_config,
1071
+ deploymentType=deployment_type,
1072
+ size=size,
1073
+ ).items() if v is not None
1074
+ }
1075
+ self._manager._patch(f'workspaces/{self.id}', json=data)
1076
+ self.refresh()
1077
+
1078
+ def refresh(self) -> Workspace:
110
1079
  """Update the object to the current state."""
111
1080
  if self._manager is None:
112
1081
  raise ManagementError(
@@ -114,7 +1083,10 @@ class Workspace(object):
114
1083
  )
115
1084
  new_obj = self._manager.get_workspace(self.id)
116
1085
  for name, value in vars(new_obj).items():
117
- setattr(self, name, value)
1086
+ if isinstance(value, Mapping):
1087
+ setattr(self, name, snake_to_camel_dict(value))
1088
+ else:
1089
+ setattr(self, name, value)
118
1090
  return self
119
1091
 
120
1092
  def terminate(
@@ -122,6 +1094,7 @@ class Workspace(object):
122
1094
  wait_on_terminated: bool = False,
123
1095
  wait_interval: int = 10,
124
1096
  wait_timeout: int = 600,
1097
+ force: bool = False,
125
1098
  ) -> None:
126
1099
  """
127
1100
  Terminate the workspace.
@@ -134,6 +1107,8 @@ class Workspace(object):
134
1107
  Number of seconds between each server check
135
1108
  wait_timeout : int, optional
136
1109
  Total number of seconds to check server before giving up
1110
+ force : bool, optional
1111
+ Should the workspace group be terminated even if it has workspaces?
137
1112
 
138
1113
  Raises
139
1114
  ------
@@ -145,7 +1120,8 @@ class Workspace(object):
145
1120
  raise ManagementError(
146
1121
  msg='No workspace manager is associated with this object.',
147
1122
  )
148
- self._manager._delete(f'workspaces/{self.id}')
1123
+ force_str = 'true' if force else 'false'
1124
+ self._manager._delete(f'workspaces/{self.id}?force={force_str}')
149
1125
  if wait_on_terminated:
150
1126
  self._manager._wait_on_state(
151
1127
  self._manager.get_workspace(self.id),
@@ -175,6 +1151,84 @@ class Workspace(object):
175
1151
  kwargs['host'] = self.endpoint
176
1152
  return connection.connect(**kwargs)
177
1153
 
1154
+ def suspend(
1155
+ self,
1156
+ wait_on_suspended: bool = False,
1157
+ wait_interval: int = 20,
1158
+ wait_timeout: int = 600,
1159
+ ) -> None:
1160
+ """
1161
+ Suspend the workspace.
1162
+
1163
+ Parameters
1164
+ ----------
1165
+ wait_on_suspended : bool, optional
1166
+ Wait for the workspace to go into 'Suspended' mode before returning
1167
+ wait_interval : int, optional
1168
+ Number of seconds between each server check
1169
+ wait_timeout : int, optional
1170
+ Total number of seconds to check server before giving up
1171
+
1172
+ Raises
1173
+ ------
1174
+ ManagementError
1175
+ If timeout is reached
1176
+
1177
+ """
1178
+ if self._manager is None:
1179
+ raise ManagementError(
1180
+ msg='No workspace manager is associated with this object.',
1181
+ )
1182
+ self._manager._post(f'workspaces/{self.id}/suspend')
1183
+ if wait_on_suspended:
1184
+ self._manager._wait_on_state(
1185
+ self._manager.get_workspace(self.id),
1186
+ 'Suspended', interval=wait_interval, timeout=wait_timeout,
1187
+ )
1188
+ self.refresh()
1189
+
1190
+ def resume(
1191
+ self,
1192
+ disable_auto_suspend: bool = False,
1193
+ wait_on_resumed: bool = False,
1194
+ wait_interval: int = 20,
1195
+ wait_timeout: int = 600,
1196
+ ) -> None:
1197
+ """
1198
+ Resume the workspace.
1199
+
1200
+ Parameters
1201
+ ----------
1202
+ disable_auto_suspend : bool, optional
1203
+ Should auto-suspend be disabled?
1204
+ wait_on_resumed : bool, optional
1205
+ Wait for the workspace to go into 'Resumed' or 'Active' mode before returning
1206
+ wait_interval : int, optional
1207
+ Number of seconds between each server check
1208
+ wait_timeout : int, optional
1209
+ Total number of seconds to check server before giving up
1210
+
1211
+ Raises
1212
+ ------
1213
+ ManagementError
1214
+ If timeout is reached
1215
+
1216
+ """
1217
+ if self._manager is None:
1218
+ raise ManagementError(
1219
+ msg='No workspace manager is associated with this object.',
1220
+ )
1221
+ self._manager._post(
1222
+ f'workspaces/{self.id}/resume',
1223
+ json=dict(disableAutoSuspend=disable_auto_suspend),
1224
+ )
1225
+ if wait_on_resumed:
1226
+ self._manager._wait_on_state(
1227
+ self._manager.get_workspace(self.id),
1228
+ ['Resumed', 'Active'], interval=wait_interval, timeout=wait_timeout,
1229
+ )
1230
+ self.refresh()
1231
+
178
1232
 
179
1233
  class WorkspaceGroup(object):
180
1234
  """
@@ -197,9 +1251,10 @@ class WorkspaceGroup(object):
197
1251
  def __init__(
198
1252
  self, name: str, id: str,
199
1253
  created_at: Union[str, datetime.datetime],
200
- region: Region,
1254
+ region: Optional[Region],
201
1255
  firewall_ranges: List[str],
202
1256
  terminated_at: Optional[Union[str, datetime.datetime]],
1257
+ allow_all_traffic: Optional[bool],
203
1258
  ):
204
1259
  #: Name of the workspace group
205
1260
  self.name = name
@@ -210,7 +1265,7 @@ class WorkspaceGroup(object):
210
1265
  #: Timestamp of when the workspace group was created
211
1266
  self.created_at = to_datetime(created_at)
212
1267
 
213
- #: Region of the cluster (see :class:`Region`)
1268
+ #: Region of the workspace group (see :class:`Region`)
214
1269
  self.region = region
215
1270
 
216
1271
  #: List of allowed incoming IP addresses / ranges
@@ -219,6 +1274,9 @@ class WorkspaceGroup(object):
219
1274
  #: Timestamp of when the workspace group was terminated
220
1275
  self.terminated_at = to_datetime(terminated_at)
221
1276
 
1277
+ #: Should all traffic be allowed?
1278
+ self.allow_all_traffic = allow_all_traffic
1279
+
222
1280
  self._manager: Optional[WorkspaceManager] = None
223
1281
 
224
1282
  def __str__(self) -> str:
@@ -248,43 +1306,88 @@ class WorkspaceGroup(object):
248
1306
  :class:`WorkspaceGroup`
249
1307
 
250
1308
  """
1309
+ try:
1310
+ region = [x for x in manager.regions if x.id == obj['regionID']][0]
1311
+ except IndexError:
1312
+ region = Region(obj.get('regionID', '<unknown>'), '<unknown>', '<unknown>')
251
1313
  out = cls(
252
- name=obj['name'], id=obj['workspaceGroupID'],
1314
+ name=obj['name'],
1315
+ id=obj['workspaceGroupID'],
253
1316
  created_at=obj['createdAt'],
254
- region=[x for x in manager.regions if x.id == obj['regionID']][0],
1317
+ region=region,
255
1318
  firewall_ranges=obj.get('firewallRanges', []),
256
1319
  terminated_at=obj.get('terminatedAt'),
1320
+ allow_all_traffic=obj.get('allowAllTraffic'),
257
1321
  )
258
1322
  out._manager = manager
259
1323
  return out
260
1324
 
1325
+ @property
1326
+ def organization(self) -> Organization:
1327
+ if self._manager is None:
1328
+ raise ManagementError(
1329
+ msg='No workspace manager is associated with this object.',
1330
+ )
1331
+ return self._manager.organization
1332
+
1333
+ @property
1334
+ def stage(self) -> Stage:
1335
+ """Stage manager."""
1336
+ if self._manager is None:
1337
+ raise ManagementError(
1338
+ msg='No workspace manager is associated with this object.',
1339
+ )
1340
+ return Stage(self, self._manager)
1341
+
1342
+ stages = stage
1343
+
261
1344
  def refresh(self) -> 'WorkspaceGroup':
262
- """Update teh object to the current state."""
1345
+ """Update the object to the current state."""
263
1346
  if self._manager is None:
264
1347
  raise ManagementError(
265
1348
  msg='No workspace manager is associated with this object.',
266
1349
  )
267
1350
  new_obj = self._manager.get_workspace_group(self.id)
268
1351
  for name, value in vars(new_obj).items():
269
- setattr(self, name, value)
1352
+ if isinstance(value, Mapping):
1353
+ setattr(self, name, camel_to_snake_dict(value))
1354
+ else:
1355
+ setattr(self, name, value)
270
1356
  return self
271
1357
 
272
1358
  def update(
273
- self, name: Optional[str] = None,
274
- admin_password: Optional[str] = None,
1359
+ self,
1360
+ name: Optional[str] = None,
275
1361
  firewall_ranges: Optional[List[str]] = None,
1362
+ admin_password: Optional[str] = None,
1363
+ expires_at: Optional[str] = None,
1364
+ allow_all_traffic: Optional[bool] = None,
1365
+ update_window: Optional[Dict[str, int]] = None,
276
1366
  ) -> None:
277
1367
  """
278
- Update the cluster definition.
1368
+ Update the workspace group definition.
279
1369
 
280
1370
  Parameters
281
1371
  ----------
282
1372
  name : str, optional
283
- Cluster name
284
- admim_password : str, optional
285
- Admin password for the cluster
286
- firewall_ranges : Sequence[str], optional
287
- List of allowed incoming IP addresses
1373
+ Name of the workspace group
1374
+ firewall_ranges : list[str], optional
1375
+ List of allowed CIDR ranges. An empty list indicates that all
1376
+ inbound requests are allowed.
1377
+ admin_password : str, optional
1378
+ Admin password for the workspace group. If no password is supplied,
1379
+ a password will be generated and retured in the response.
1380
+ expires_at : str, optional
1381
+ The timestamp of when the workspace group will expire.
1382
+ If the expiration time is not specified,
1383
+ the workspace group will have no expiration time.
1384
+ At expiration, the workspace group is terminated and all the data is lost.
1385
+ Expiration time can be specified as a timestamp or duration.
1386
+ Example: "2021-01-02T15:04:05Z07:00", "2021-01-02", "3h30m"
1387
+ allow_all_traffic : bool, optional
1388
+ Allow all traffic to the workspace group
1389
+ update_window : Dict[str, int], optional
1390
+ Specify the day and hour of an update window: dict(day=0-6, hour=0-23)
288
1391
 
289
1392
  """
290
1393
  if self._manager is None:
@@ -293,8 +1396,12 @@ class WorkspaceGroup(object):
293
1396
  )
294
1397
  data = {
295
1398
  k: v for k, v in dict(
296
- name=name, adminPassword=admin_password,
1399
+ name=name,
297
1400
  firewallRanges=firewall_ranges,
1401
+ adminPassword=admin_password,
1402
+ expiresAt=expires_at,
1403
+ allowAllTraffic=allow_all_traffic,
1404
+ updateWindow=snake_to_camel_dict(update_window),
298
1405
  ).items() if v is not None
299
1406
  }
300
1407
  self._manager._patch(f'workspaceGroups/{self.id}', json=data)
@@ -314,7 +1421,7 @@ class WorkspaceGroup(object):
314
1421
  force : bool, optional
315
1422
  Terminate a workspace group even if it has active workspaces
316
1423
  wait_on_terminated : bool, optional
317
- Wait for the cluster to go into 'Terminated' mode before returning
1424
+ Wait for the workspace group to go into 'Terminated' mode before returning
318
1425
  wait_interval : int, optional
319
1426
  Number of seconds between each server check
320
1427
  wait_timeout : int, optional
@@ -332,15 +1439,26 @@ class WorkspaceGroup(object):
332
1439
  )
333
1440
  self._manager._delete(f'workspaceGroups/{self.id}', params=dict(force=force))
334
1441
  if wait_on_terminated:
335
- self._manager._wait_on_state(
336
- self._manager.get_workspace_group(self.id),
337
- 'Terminated', interval=wait_interval, timeout=wait_timeout,
338
- )
339
- self.refresh()
1442
+ while True:
1443
+ self.refresh()
1444
+ if self.terminated_at is not None:
1445
+ break
1446
+ if wait_timeout <= 0:
1447
+ raise ManagementError(
1448
+ msg='Exceeded waiting time for WorkspaceGroup to terminate',
1449
+ )
1450
+ time.sleep(wait_interval)
1451
+ wait_timeout -= wait_interval
340
1452
 
341
1453
  def create_workspace(
342
- self, name: str, size: Optional[str] = None,
343
- wait_on_active: bool = False, wait_interval: int = 10,
1454
+ self,
1455
+ name: str,
1456
+ size: Optional[str] = None,
1457
+ auto_suspend: Optional[Dict[str, Any]] = None,
1458
+ cache_config: Optional[int] = None,
1459
+ enable_kai: Optional[bool] = None,
1460
+ wait_on_active: bool = False,
1461
+ wait_interval: int = 10,
344
1462
  wait_timeout: int = 600,
345
1463
  ) -> Workspace:
346
1464
  """
@@ -352,6 +1470,15 @@ class WorkspaceGroup(object):
352
1470
  Name of the workspace
353
1471
  size : str, optional
354
1472
  Workspace size in workspace size notation (S-00, S-1, etc.)
1473
+ auto_suspend : Dict[str, Any], optional
1474
+ Auto suspend settings for the workspace. If this field is not
1475
+ provided, no settings will be enabled.
1476
+ cache_config : int, optional
1477
+ Specifies the multiplier for the persistent cache associated
1478
+ with the workspace. If specified, it enables the cache configuration
1479
+ multiplier. It can have one of the following values: 1, 2, or 4.
1480
+ enable_kai : bool, optional
1481
+ Whether to create a SingleStore Kai-enabled workspace
355
1482
  wait_on_active : bool, optional
356
1483
  Wait for the workspace to be active before returning
357
1484
  wait_timeout : int, optional
@@ -369,20 +1496,103 @@ class WorkspaceGroup(object):
369
1496
  raise ManagementError(
370
1497
  msg='No workspace manager is associated with this object.',
371
1498
  )
372
- return self._manager.create_workspace(
373
- name=name, workspace_group=self, size=size, wait_on_active=wait_on_active,
374
- wait_interval=wait_interval, wait_timeout=wait_timeout,
1499
+
1500
+ out = self._manager.create_workspace(
1501
+ name=name,
1502
+ workspace_group=self,
1503
+ size=size,
1504
+ auto_suspend=snake_to_camel_dict(auto_suspend),
1505
+ cache_config=cache_config,
1506
+ enable_kai=enable_kai,
1507
+ wait_on_active=wait_on_active,
1508
+ wait_interval=wait_interval,
1509
+ wait_timeout=wait_timeout,
375
1510
  )
376
1511
 
1512
+ return out
1513
+
377
1514
  @property
378
- def workspaces(self) -> List[Workspace]:
1515
+ def workspaces(self) -> NamedList[Workspace]:
379
1516
  """Return a list of available workspaces."""
380
1517
  if self._manager is None:
381
1518
  raise ManagementError(
382
1519
  msg='No workspace manager is associated with this object.',
383
1520
  )
384
1521
  res = self._manager._get('workspaces', params=dict(workspaceGroupID=self.id))
385
- return [Workspace.from_dict(item, self._manager) for item in res.json()]
1522
+ return NamedList(
1523
+ [Workspace.from_dict(item, self._manager) for item in res.json()],
1524
+ )
1525
+
1526
+
1527
+ class Billing(object):
1528
+ """Billing information."""
1529
+
1530
+ COMPUTE_CREDIT = 'compute_credit'
1531
+ STORAGE_AVG_BYTE = 'storage_avg_byte'
1532
+
1533
+ HOUR = 'hour'
1534
+ DAY = 'day'
1535
+ MONTH = 'month'
1536
+
1537
+ def __init__(self, manager: Manager):
1538
+ self._manager = manager
1539
+
1540
+ def usage(
1541
+ self,
1542
+ start_time: datetime.datetime,
1543
+ end_time: datetime.datetime,
1544
+ metric: Optional[str] = None,
1545
+ aggregate_by: Optional[str] = None,
1546
+ ) -> List[BillingUsageItem]:
1547
+ """
1548
+ Get usage information.
1549
+
1550
+ Parameters
1551
+ ----------
1552
+ start_time : datetime.datetime
1553
+ Start time for usage interval
1554
+ end_time : datetime.datetime
1555
+ End time for usage interval
1556
+ metric : str, optional
1557
+ Possible metrics are ``mgr.billing.COMPUTE_CREDIT`` and
1558
+ ``mgr.billing.STORAGE_AVG_BYTE`` (default is all)
1559
+ aggregate_by : str, optional
1560
+ Aggregate type used to group usage: ``mgr.billing.HOUR``,
1561
+ ``mgr.billing.DAY``, or ``mgr.billing.MONTH``
1562
+
1563
+ Returns
1564
+ -------
1565
+ List[BillingUsage]
1566
+
1567
+ """
1568
+ res = self._manager._get(
1569
+ 'billing/usage',
1570
+ params={
1571
+ k: v for k, v in dict(
1572
+ metric=snake_to_camel(metric),
1573
+ startTime=from_datetime(start_time),
1574
+ endTime=from_datetime(end_time),
1575
+ aggregate_by=aggregate_by.lower() if aggregate_by else None,
1576
+ ).items() if v is not None
1577
+ },
1578
+ )
1579
+ return [
1580
+ BillingUsageItem.from_dict(x, self._manager)
1581
+ for x in res.json()['billingUsage']
1582
+ ]
1583
+
1584
+
1585
+ class Organizations(object):
1586
+ """Organizations."""
1587
+
1588
+ def __init__(self, manager: Manager):
1589
+ self._manager = manager
1590
+
1591
+ @property
1592
+ def current(self) -> Organization:
1593
+ """Get current organization."""
1594
+ res = self._manager._get('organizations/current').json()
1595
+ return Organization.from_dict(res, self._manager)
386
1596
 
387
1597
 
388
1598
  class WorkspaceManager(Manager):
@@ -416,20 +1626,44 @@ class WorkspaceManager(Manager):
416
1626
  obj_type = 'workspace'
417
1627
 
418
1628
  @property
419
- def workspace_groups(self) -> List[WorkspaceGroup]:
1629
+ def workspace_groups(self) -> NamedList[WorkspaceGroup]:
420
1630
  """Return a list of available workspace groups."""
421
1631
  res = self._get('workspaceGroups')
422
- return [WorkspaceGroup.from_dict(item, self) for item in res.json()]
1632
+ return NamedList([WorkspaceGroup.from_dict(item, self) for item in res.json()])
1633
+
1634
+ @property
1635
+ def organizations(self) -> Organizations:
1636
+ """Return the organizations."""
1637
+ return Organizations(self)
1638
+
1639
+ @property
1640
+ def organization(self) -> Organization:
1641
+ """ Return the current organization."""
1642
+ return self.organizations.current
423
1643
 
424
1644
  @property
425
- def regions(self) -> List[Region]:
1645
+ def billing(self) -> Billing:
1646
+ """Return the current billing information."""
1647
+ return Billing(self)
1648
+
1649
+ @ttl_property(datetime.timedelta(hours=1))
1650
+ def regions(self) -> NamedList[Region]:
426
1651
  """Return a list of available regions."""
427
1652
  res = self._get('regions')
428
- return [Region.from_dict(item, self) for item in res.json()]
1653
+ return NamedList([Region.from_dict(item, self) for item in res.json()])
429
1654
 
430
1655
  def create_workspace_group(
431
- self, name: str, region: Union[str, Region],
432
- firewall_ranges: List[str], admin_password: Optional[str] = None,
1656
+ self,
1657
+ name: str,
1658
+ region: Union[str, Region],
1659
+ firewall_ranges: List[str],
1660
+ admin_password: Optional[str] = None,
1661
+ backup_bucket_kms_key_id: Optional[str] = None,
1662
+ data_bucket_kms_key_id: Optional[str] = None,
1663
+ expires_at: Optional[str] = None,
1664
+ smart_dr: Optional[bool] = None,
1665
+ allow_all_traffic: Optional[bool] = None,
1666
+ update_window: Optional[Dict[str, int]] = None,
433
1667
  ) -> WorkspaceGroup:
434
1668
  """
435
1669
  Create a new workspace group.
@@ -446,6 +1680,32 @@ class WorkspaceManager(Manager):
446
1680
  admin_password : str, optional
447
1681
  Admin password for the workspace group. If no password is supplied,
448
1682
  a password will be generated and retured in the response.
1683
+ backup_bucket_kms_key_id : str, optional
1684
+ Specifies the KMS key ID associated with the backup bucket.
1685
+ If specified, enables Customer-Managed Encryption Keys (CMEK)
1686
+ encryption for the backup bucket of the workspace group.
1687
+ This feature is only supported in workspace groups deployed in AWS.
1688
+ data_bucket_kms_key_id : str, optional
1689
+ Specifies the KMS key ID associated with the data bucket.
1690
+ If specified, enables Customer-Managed Encryption Keys (CMEK)
1691
+ encryption for the data bucket and Amazon Elastic Block Store
1692
+ (EBS) volumes of the workspace group. This feature is only supported
1693
+ in workspace groups deployed in AWS.
1694
+ expires_at : str, optional
1695
+ The timestamp of when the workspace group will expire.
1696
+ If the expiration time is not specified,
1697
+ the workspace group will have no expiration time.
1698
+ At expiration, the workspace group is terminated and all the data is lost.
1699
+ Expiration time can be specified as a timestamp or duration.
1700
+ Example: "2021-01-02T15:04:05Z07:00", "2021-01-02", "3h30m"
1701
+ smart_dr : bool, optional
1702
+ Enables Smart Disaster Recovery (SmartDR) for the workspace group.
1703
+ SmartDR is a disaster recovery solution that ensures seamless and
1704
+ continuous replication of data from the primary region to a secondary region
1705
+ allow_all_traffic : bool, optional
1706
+ Allow all traffic to the workspace group
1707
+ update_window : Dict[str, int], optional
1708
+ Specify the day and hour of an update window: dict(day=0-6, hour=0-23)
449
1709
 
450
1710
  Returns
451
1711
  -------
@@ -458,15 +1718,28 @@ class WorkspaceManager(Manager):
458
1718
  'workspaceGroups', json=dict(
459
1719
  name=name, regionID=region,
460
1720
  adminPassword=admin_password,
461
- firewallRanges=firewall_ranges,
1721
+ backupBucketKMSKeyID=backup_bucket_kms_key_id,
1722
+ dataBucketKMSKeyID=data_bucket_kms_key_id,
1723
+ firewallRanges=firewall_ranges or [],
1724
+ expiresAt=expires_at,
1725
+ smartDR=smart_dr,
1726
+ allowAllTraffic=allow_all_traffic,
1727
+ updateWindow=snake_to_camel_dict(update_window),
462
1728
  ),
463
1729
  )
464
1730
  return self.get_workspace_group(res.json()['workspaceGroupID'])
465
1731
 
466
1732
  def create_workspace(
467
- self, name: str, workspace_group: Union[str, WorkspaceGroup],
468
- size: Optional[str] = None, wait_on_active: bool = False,
469
- wait_interval: int = 10, wait_timeout: int = 600,
1733
+ self,
1734
+ name: str,
1735
+ workspace_group: Union[str, WorkspaceGroup],
1736
+ size: Optional[str] = None,
1737
+ auto_suspend: Optional[Dict[str, Any]] = None,
1738
+ cache_config: Optional[int] = None,
1739
+ enable_kai: Optional[bool] = None,
1740
+ wait_on_active: bool = False,
1741
+ wait_interval: int = 10,
1742
+ wait_timeout: int = 600,
470
1743
  ) -> Workspace:
471
1744
  """
472
1745
  Create a new workspace.
@@ -479,6 +1752,15 @@ class WorkspaceManager(Manager):
479
1752
  The workspace ID of the workspace
480
1753
  size : str, optional
481
1754
  Workspace size in workspace size notation (S-00, S-1, etc.)
1755
+ auto_suspend : Dict[str, Any], optional
1756
+ Auto suspend settings for the workspace. If this field is not
1757
+ provided, no settings will be enabled.
1758
+ cache_config : int, optional
1759
+ Specifies the multiplier for the persistent cache associated
1760
+ with the workspace. If specified, it enables the cache configuration
1761
+ multiplier. It can have one of the following values: 1, 2, or 4.
1762
+ enable_kai : bool, optional
1763
+ Whether to create a SingleStore Kai-enabled workspace
482
1764
  wait_on_active : bool, optional
483
1765
  Wait for the workspace to be active before returning
484
1766
  wait_timeout : int, optional
@@ -496,14 +1778,20 @@ class WorkspaceManager(Manager):
496
1778
  workspace_group = workspace_group.id
497
1779
  res = self._post(
498
1780
  'workspaces', json=dict(
499
- name=name, workspaceGroupID=workspace_group,
1781
+ name=name,
1782
+ workspaceGroupID=workspace_group,
500
1783
  size=size,
1784
+ autoSuspend=snake_to_camel_dict(auto_suspend),
1785
+ cacheConfig=cache_config,
1786
+ enableKai=enable_kai,
501
1787
  ),
502
1788
  )
503
1789
  out = self.get_workspace(res.json()['workspaceID'])
504
1790
  if wait_on_active:
505
1791
  out = self._wait_on_state(
506
- out, 'Active', interval=wait_interval,
1792
+ out,
1793
+ 'Active',
1794
+ interval=wait_interval,
507
1795
  timeout=wait_timeout,
508
1796
  )
509
1797
  return out
@@ -547,6 +1835,8 @@ def manage_workspaces(
547
1835
  access_token: Optional[str] = None,
548
1836
  version: str = WorkspaceManager.default_version,
549
1837
  base_url: str = WorkspaceManager.default_base_url,
1838
+ *,
1839
+ organization_id: Optional[str] = None,
550
1840
  ) -> WorkspaceManager:
551
1841
  """
552
1842
  Retrieve a SingleStoreDB workspace manager.
@@ -559,10 +1849,15 @@ def manage_workspaces(
559
1849
  Version of the API to use
560
1850
  base_url : str, optional
561
1851
  Base URL of the workspace management API
1852
+ organization_id : str, optional
1853
+ ID of organization, if using a JWT for authentication
562
1854
 
563
1855
  Returns
564
1856
  -------
565
1857
  :class:`WorkspaceManager`
566
1858
 
567
1859
  """
568
- return WorkspaceManager(access_token=access_token, base_url=base_url, version=version)
1860
+ return WorkspaceManager(
1861
+ access_token=access_token, base_url=base_url,
1862
+ version=version, organization_id=organization_id,
1863
+ )