singlestoredb 0.4.0__py3-none-any.whl → 1.0.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of singlestoredb might be problematic. Click here for more details.
- singlestoredb/__init__.py +33 -1
- singlestoredb/alchemy/__init__.py +90 -0
- singlestoredb/auth.py +5 -1
- singlestoredb/config.py +116 -14
- singlestoredb/connection.py +483 -516
- singlestoredb/converters.py +238 -135
- singlestoredb/exceptions.py +30 -2
- singlestoredb/functions/__init__.py +1 -0
- singlestoredb/functions/decorator.py +142 -0
- singlestoredb/functions/dtypes.py +1639 -0
- singlestoredb/functions/ext/__init__.py +2 -0
- singlestoredb/functions/ext/arrow.py +375 -0
- singlestoredb/functions/ext/asgi.py +661 -0
- singlestoredb/functions/ext/json.py +427 -0
- singlestoredb/functions/ext/mmap.py +306 -0
- singlestoredb/functions/ext/rowdat_1.py +744 -0
- singlestoredb/functions/signature.py +673 -0
- singlestoredb/fusion/__init__.py +11 -0
- singlestoredb/fusion/graphql.py +213 -0
- singlestoredb/fusion/handler.py +621 -0
- singlestoredb/fusion/handlers/stage.py +257 -0
- singlestoredb/fusion/handlers/utils.py +162 -0
- singlestoredb/fusion/handlers/workspace.py +412 -0
- singlestoredb/fusion/registry.py +164 -0
- singlestoredb/fusion/result.py +399 -0
- singlestoredb/http/__init__.py +27 -0
- singlestoredb/{http.py → http/connection.py} +555 -154
- singlestoredb/management/__init__.py +3 -0
- singlestoredb/management/billing_usage.py +148 -0
- singlestoredb/management/cluster.py +14 -6
- singlestoredb/management/manager.py +100 -38
- singlestoredb/management/organization.py +188 -0
- singlestoredb/management/region.py +5 -5
- singlestoredb/management/utils.py +281 -2
- singlestoredb/management/workspace.py +1344 -49
- singlestoredb/{clients/pymysqlsv → mysql}/__init__.py +16 -21
- singlestoredb/{clients/pymysqlsv → mysql}/_auth.py +39 -8
- singlestoredb/{clients/pymysqlsv → mysql}/charset.py +26 -23
- singlestoredb/{clients/pymysqlsv/connections.py → mysql/connection.py} +532 -165
- singlestoredb/{clients/pymysqlsv → mysql}/constants/CLIENT.py +0 -1
- singlestoredb/{clients/pymysqlsv → mysql}/constants/COMMAND.py +0 -1
- singlestoredb/{clients/pymysqlsv → mysql}/constants/CR.py +0 -2
- singlestoredb/{clients/pymysqlsv → mysql}/constants/ER.py +0 -1
- singlestoredb/{clients/pymysqlsv → mysql}/constants/FIELD_TYPE.py +1 -1
- singlestoredb/{clients/pymysqlsv → mysql}/constants/FLAG.py +0 -1
- singlestoredb/{clients/pymysqlsv → mysql}/constants/SERVER_STATUS.py +0 -1
- singlestoredb/mysql/converters.py +271 -0
- singlestoredb/{clients/pymysqlsv → mysql}/cursors.py +228 -112
- singlestoredb/mysql/err.py +92 -0
- singlestoredb/{clients/pymysqlsv → mysql}/optionfile.py +5 -4
- singlestoredb/{clients/pymysqlsv → mysql}/protocol.py +49 -20
- singlestoredb/mysql/tests/__init__.py +19 -0
- singlestoredb/{clients/pymysqlsv → mysql}/tests/base.py +32 -12
- singlestoredb/mysql/tests/conftest.py +37 -0
- singlestoredb/{clients/pymysqlsv → mysql}/tests/test_DictCursor.py +11 -7
- singlestoredb/{clients/pymysqlsv → mysql}/tests/test_SSCursor.py +17 -12
- singlestoredb/{clients/pymysqlsv → mysql}/tests/test_basic.py +32 -24
- singlestoredb/{clients/pymysqlsv → mysql}/tests/test_connection.py +130 -119
- singlestoredb/{clients/pymysqlsv → mysql}/tests/test_converters.py +9 -7
- singlestoredb/mysql/tests/test_cursor.py +141 -0
- singlestoredb/{clients/pymysqlsv → mysql}/tests/test_err.py +3 -2
- singlestoredb/{clients/pymysqlsv → mysql}/tests/test_issues.py +35 -27
- singlestoredb/{clients/pymysqlsv → mysql}/tests/test_load_local.py +13 -11
- singlestoredb/{clients/pymysqlsv → mysql}/tests/test_nextset.py +7 -3
- singlestoredb/{clients/pymysqlsv → mysql}/tests/test_optionfile.py +2 -1
- singlestoredb/{clients/pymysqlsv → mysql}/tests/thirdparty/__init__.py +1 -1
- singlestoredb/mysql/tests/thirdparty/test_MySQLdb/__init__.py +9 -0
- singlestoredb/{clients/pymysqlsv → mysql}/tests/thirdparty/test_MySQLdb/capabilities.py +19 -17
- singlestoredb/{clients/pymysqlsv → mysql}/tests/thirdparty/test_MySQLdb/dbapi20.py +31 -22
- singlestoredb/{clients/pymysqlsv → mysql}/tests/thirdparty/test_MySQLdb/test_MySQLdb_capabilities.py +3 -4
- singlestoredb/{clients/pymysqlsv → mysql}/tests/thirdparty/test_MySQLdb/test_MySQLdb_dbapi20.py +24 -20
- singlestoredb/{clients/pymysqlsv → mysql}/tests/thirdparty/test_MySQLdb/test_MySQLdb_nonstandard.py +4 -4
- singlestoredb/{clients/pymysqlsv → mysql}/times.py +3 -4
- singlestoredb/pytest.py +283 -0
- singlestoredb/tests/empty.sql +0 -0
- singlestoredb/tests/ext_funcs/__init__.py +385 -0
- singlestoredb/tests/test.sql +210 -0
- singlestoredb/tests/test2.sql +1 -0
- singlestoredb/tests/test_basics.py +482 -115
- singlestoredb/tests/test_config.py +13 -13
- singlestoredb/tests/test_connection.py +241 -305
- singlestoredb/tests/test_dbapi.py +27 -0
- singlestoredb/tests/test_ext_func.py +1193 -0
- singlestoredb/tests/test_ext_func_data.py +1101 -0
- singlestoredb/tests/test_fusion.py +465 -0
- singlestoredb/tests/test_http.py +32 -26
- singlestoredb/tests/test_management.py +588 -8
- singlestoredb/tests/test_plugin.py +33 -0
- singlestoredb/tests/test_results.py +11 -12
- singlestoredb/tests/test_udf.py +687 -0
- singlestoredb/tests/utils.py +3 -2
- singlestoredb/utils/config.py +58 -0
- singlestoredb/utils/debug.py +13 -0
- singlestoredb/utils/mogrify.py +151 -0
- singlestoredb/utils/results.py +4 -1
- singlestoredb-1.0.4.dist-info/METADATA +139 -0
- singlestoredb-1.0.4.dist-info/RECORD +112 -0
- {singlestoredb-0.4.0.dist-info → singlestoredb-1.0.4.dist-info}/WHEEL +1 -1
- singlestoredb-1.0.4.dist-info/entry_points.txt +2 -0
- singlestoredb/clients/pymysqlsv/converters.py +0 -365
- singlestoredb/clients/pymysqlsv/err.py +0 -144
- singlestoredb/clients/pymysqlsv/tests/__init__.py +0 -19
- singlestoredb/clients/pymysqlsv/tests/test_cursor.py +0 -133
- singlestoredb/clients/pymysqlsv/tests/thirdparty/test_MySQLdb/__init__.py +0 -9
- singlestoredb/drivers/__init__.py +0 -45
- singlestoredb/drivers/base.py +0 -198
- singlestoredb/drivers/cymysql.py +0 -38
- singlestoredb/drivers/http.py +0 -47
- singlestoredb/drivers/mariadb.py +0 -40
- singlestoredb/drivers/mysqlconnector.py +0 -49
- singlestoredb/drivers/mysqldb.py +0 -60
- singlestoredb/drivers/pymysql.py +0 -37
- singlestoredb/drivers/pymysqlsv.py +0 -35
- singlestoredb/drivers/pyodbc.py +0 -65
- singlestoredb-0.4.0.dist-info/METADATA +0 -111
- singlestoredb-0.4.0.dist-info/RECORD +0 -86
- /singlestoredb/{clients → fusion/handlers}/__init__.py +0 -0
- /singlestoredb/{clients/pymysqlsv → mysql}/constants/__init__.py +0 -0
- {singlestoredb-0.4.0.dist-info → singlestoredb-1.0.4.dist-info}/LICENSE +0 -0
- {singlestoredb-0.4.0.dist-info → singlestoredb-1.0.4.dist-info}/top_level.txt +0 -0
|
@@ -1,20 +1,913 @@
|
|
|
1
1
|
#!/usr/bin/env python
|
|
2
2
|
"""SingleStoreDB Workspace Management."""
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
3
5
|
import datetime
|
|
6
|
+
import glob
|
|
7
|
+
import io
|
|
8
|
+
import os
|
|
9
|
+
import re
|
|
10
|
+
import time
|
|
11
|
+
from collections.abc import Mapping
|
|
4
12
|
from typing import Any
|
|
13
|
+
from typing import BinaryIO
|
|
5
14
|
from typing import Dict
|
|
6
15
|
from typing import List
|
|
7
16
|
from typing import Optional
|
|
17
|
+
from typing import TextIO
|
|
8
18
|
from typing import Union
|
|
9
19
|
|
|
10
20
|
from .. import connection
|
|
11
21
|
from ..exceptions import ManagementError
|
|
22
|
+
from .billing_usage import BillingUsageItem
|
|
12
23
|
from .manager import Manager
|
|
24
|
+
from .organization import Organization
|
|
13
25
|
from .region import Region
|
|
26
|
+
from .utils import camel_to_snake_dict
|
|
27
|
+
from .utils import from_datetime
|
|
28
|
+
from .utils import NamedList
|
|
29
|
+
from .utils import PathLike
|
|
30
|
+
from .utils import snake_to_camel
|
|
31
|
+
from .utils import snake_to_camel_dict
|
|
14
32
|
from .utils import to_datetime
|
|
33
|
+
from .utils import ttl_property
|
|
15
34
|
from .utils import vars_to_str
|
|
16
35
|
|
|
17
36
|
|
|
37
|
+
def get_secret(name: str) -> str:
|
|
38
|
+
"""Get a secret from the organization."""
|
|
39
|
+
return manage_workspaces().organization.get_secret(name).value
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
class StageObject(object):
|
|
43
|
+
"""
|
|
44
|
+
Stage file / folder object.
|
|
45
|
+
|
|
46
|
+
This object is not instantiated directly. It is used in the results
|
|
47
|
+
of various operations in ``WorkspaceGroup.stage`` methods.
|
|
48
|
+
|
|
49
|
+
"""
|
|
50
|
+
|
|
51
|
+
def __init__(
|
|
52
|
+
self,
|
|
53
|
+
name: str,
|
|
54
|
+
path: str,
|
|
55
|
+
size: int,
|
|
56
|
+
type: str,
|
|
57
|
+
format: str,
|
|
58
|
+
mimetype: str,
|
|
59
|
+
created: Optional[datetime.datetime],
|
|
60
|
+
last_modified: Optional[datetime.datetime],
|
|
61
|
+
writable: bool,
|
|
62
|
+
content: Optional[List[str]] = None,
|
|
63
|
+
):
|
|
64
|
+
#: Name of file / folder
|
|
65
|
+
self.name = name
|
|
66
|
+
|
|
67
|
+
if type == 'directory':
|
|
68
|
+
path = re.sub(r'/*$', r'', str(path)) + '/'
|
|
69
|
+
|
|
70
|
+
#: Path of file / folder
|
|
71
|
+
self.path = path
|
|
72
|
+
|
|
73
|
+
#: Size of the object (in bytes)
|
|
74
|
+
self.size = size
|
|
75
|
+
|
|
76
|
+
#: Data type: file or directory
|
|
77
|
+
self.type = type
|
|
78
|
+
|
|
79
|
+
#: Data format
|
|
80
|
+
self.format = format
|
|
81
|
+
|
|
82
|
+
#: Mime type
|
|
83
|
+
self.mimetype = mimetype
|
|
84
|
+
|
|
85
|
+
#: Datetime the object was created
|
|
86
|
+
self.created_at = created
|
|
87
|
+
|
|
88
|
+
#: Datetime the object was modified last
|
|
89
|
+
self.last_modified_at = last_modified
|
|
90
|
+
|
|
91
|
+
#: Is the object writable?
|
|
92
|
+
self.writable = writable
|
|
93
|
+
|
|
94
|
+
#: Contents of a directory
|
|
95
|
+
self.content: List[str] = content or []
|
|
96
|
+
|
|
97
|
+
self._stage: Optional[Stage] = None
|
|
98
|
+
|
|
99
|
+
@classmethod
|
|
100
|
+
def from_dict(
|
|
101
|
+
cls,
|
|
102
|
+
obj: Dict[str, Any],
|
|
103
|
+
stage: Stage,
|
|
104
|
+
) -> StageObject:
|
|
105
|
+
"""
|
|
106
|
+
Construct a StageObject from a dictionary of values.
|
|
107
|
+
|
|
108
|
+
Parameters
|
|
109
|
+
----------
|
|
110
|
+
obj : dict
|
|
111
|
+
Dictionary of values
|
|
112
|
+
stage : Stage
|
|
113
|
+
Stage object to use as the parent
|
|
114
|
+
|
|
115
|
+
Returns
|
|
116
|
+
-------
|
|
117
|
+
:class:`StageObject`
|
|
118
|
+
|
|
119
|
+
"""
|
|
120
|
+
out = cls(
|
|
121
|
+
name=obj['name'],
|
|
122
|
+
path=obj['path'],
|
|
123
|
+
size=obj['size'],
|
|
124
|
+
type=obj['type'],
|
|
125
|
+
format=obj['format'],
|
|
126
|
+
mimetype=obj['mimetype'],
|
|
127
|
+
created=to_datetime(obj.get('created')),
|
|
128
|
+
last_modified=to_datetime(obj.get('last_modified')),
|
|
129
|
+
writable=bool(obj['writable']),
|
|
130
|
+
)
|
|
131
|
+
out._stage = stage
|
|
132
|
+
return out
|
|
133
|
+
|
|
134
|
+
def __str__(self) -> str:
|
|
135
|
+
"""Return string representation."""
|
|
136
|
+
return vars_to_str(self)
|
|
137
|
+
|
|
138
|
+
def __repr__(self) -> str:
|
|
139
|
+
"""Return string representation."""
|
|
140
|
+
return str(self)
|
|
141
|
+
|
|
142
|
+
def open(
|
|
143
|
+
self,
|
|
144
|
+
mode: str = 'r',
|
|
145
|
+
encoding: Optional[str] = None,
|
|
146
|
+
) -> Union[io.StringIO, io.BytesIO]:
|
|
147
|
+
"""
|
|
148
|
+
Open a Stage path for reading or writing.
|
|
149
|
+
|
|
150
|
+
Parameters
|
|
151
|
+
----------
|
|
152
|
+
mode : str, optional
|
|
153
|
+
The read / write mode. The following modes are supported:
|
|
154
|
+
* 'r' open for reading (default)
|
|
155
|
+
* 'w' open for writing, truncating the file first
|
|
156
|
+
* 'x' create a new file and open it for writing
|
|
157
|
+
The data type can be specified by adding one of the following:
|
|
158
|
+
* 'b' binary mode
|
|
159
|
+
* 't' text mode (default)
|
|
160
|
+
encoding : str, optional
|
|
161
|
+
The string encoding to use for text
|
|
162
|
+
|
|
163
|
+
Returns
|
|
164
|
+
-------
|
|
165
|
+
StageObjectBytesReader - 'rb' or 'b' mode
|
|
166
|
+
StageObjectBytesWriter - 'wb' or 'xb' mode
|
|
167
|
+
StageObjectTextReader - 'r' or 'rt' mode
|
|
168
|
+
StageObjectTextWriter - 'w', 'x', 'wt' or 'xt' mode
|
|
169
|
+
|
|
170
|
+
"""
|
|
171
|
+
if self._stage is None:
|
|
172
|
+
raise ManagementError(
|
|
173
|
+
msg='No Stage object is associated with this object.',
|
|
174
|
+
)
|
|
175
|
+
|
|
176
|
+
if self.is_dir():
|
|
177
|
+
raise IsADirectoryError(
|
|
178
|
+
f'directories can not be read or written: {self.path}',
|
|
179
|
+
)
|
|
180
|
+
|
|
181
|
+
return self._stage.open(self.path, mode=mode, encoding=encoding)
|
|
182
|
+
|
|
183
|
+
def download(
|
|
184
|
+
self,
|
|
185
|
+
local_path: Optional[PathLike] = None,
|
|
186
|
+
*,
|
|
187
|
+
overwrite: bool = False,
|
|
188
|
+
encoding: Optional[str] = None,
|
|
189
|
+
) -> Optional[Union[bytes, str]]:
|
|
190
|
+
"""
|
|
191
|
+
Download the content of a stage path.
|
|
192
|
+
|
|
193
|
+
Parameters
|
|
194
|
+
----------
|
|
195
|
+
local_path : Path or str
|
|
196
|
+
Path to local file target location
|
|
197
|
+
overwrite : bool, optional
|
|
198
|
+
Should an existing file be overwritten if it exists?
|
|
199
|
+
encoding : str, optional
|
|
200
|
+
Encoding used to convert the resulting data
|
|
201
|
+
|
|
202
|
+
Returns
|
|
203
|
+
-------
|
|
204
|
+
bytes or str or None
|
|
205
|
+
|
|
206
|
+
"""
|
|
207
|
+
if self._stage is None:
|
|
208
|
+
raise ManagementError(
|
|
209
|
+
msg='No Stage object is associated with this object.',
|
|
210
|
+
)
|
|
211
|
+
|
|
212
|
+
return self._stage.download_file(
|
|
213
|
+
self.path, local_path=local_path,
|
|
214
|
+
overwrite=overwrite, encoding=encoding,
|
|
215
|
+
)
|
|
216
|
+
|
|
217
|
+
download_file = download
|
|
218
|
+
|
|
219
|
+
def remove(self) -> None:
|
|
220
|
+
"""Delete the stage file."""
|
|
221
|
+
if self._stage is None:
|
|
222
|
+
raise ManagementError(
|
|
223
|
+
msg='No Stage object is associated with this object.',
|
|
224
|
+
)
|
|
225
|
+
|
|
226
|
+
if self.type == 'directory':
|
|
227
|
+
raise IsADirectoryError(
|
|
228
|
+
f'path is a directory; use rmdir or removedirs {self.path}',
|
|
229
|
+
)
|
|
230
|
+
|
|
231
|
+
self._stage.remove(self.path)
|
|
232
|
+
|
|
233
|
+
def rmdir(self) -> None:
|
|
234
|
+
"""Delete the empty stage directory."""
|
|
235
|
+
if self._stage is None:
|
|
236
|
+
raise ManagementError(
|
|
237
|
+
msg='No Stage object is associated with this object.',
|
|
238
|
+
)
|
|
239
|
+
|
|
240
|
+
if self.type != 'directory':
|
|
241
|
+
raise NotADirectoryError(
|
|
242
|
+
f'path is not a directory: {self.path}',
|
|
243
|
+
)
|
|
244
|
+
|
|
245
|
+
self._stage.rmdir(self.path)
|
|
246
|
+
|
|
247
|
+
def removedirs(self) -> None:
|
|
248
|
+
"""Delete the stage directory recursively."""
|
|
249
|
+
if self._stage is None:
|
|
250
|
+
raise ManagementError(
|
|
251
|
+
msg='No Stage object is associated with this object.',
|
|
252
|
+
)
|
|
253
|
+
|
|
254
|
+
if self.type != 'directory':
|
|
255
|
+
raise NotADirectoryError(
|
|
256
|
+
f'path is not a directory: {self.path}',
|
|
257
|
+
)
|
|
258
|
+
|
|
259
|
+
self._stage.removedirs(self.path)
|
|
260
|
+
|
|
261
|
+
def rename(self, new_path: PathLike, *, overwrite: bool = False) -> None:
|
|
262
|
+
"""
|
|
263
|
+
Move the stage file to a new location.
|
|
264
|
+
|
|
265
|
+
Parameters
|
|
266
|
+
----------
|
|
267
|
+
new_path : Path or str
|
|
268
|
+
The new location of the file
|
|
269
|
+
overwrite : bool, optional
|
|
270
|
+
Should path be overwritten if it already exists?
|
|
271
|
+
|
|
272
|
+
"""
|
|
273
|
+
if self._stage is None:
|
|
274
|
+
raise ManagementError(
|
|
275
|
+
msg='No Stage object is associated with this object.',
|
|
276
|
+
)
|
|
277
|
+
out = self._stage.rename(self.path, new_path, overwrite=overwrite)
|
|
278
|
+
self.name = out.name
|
|
279
|
+
self.path = out.path
|
|
280
|
+
return None
|
|
281
|
+
|
|
282
|
+
def exists(self) -> bool:
|
|
283
|
+
"""Does the file / folder exist?"""
|
|
284
|
+
if self._stage is None:
|
|
285
|
+
raise ManagementError(
|
|
286
|
+
msg='No Stage object is associated with this object.',
|
|
287
|
+
)
|
|
288
|
+
return self._stage.exists(self.path)
|
|
289
|
+
|
|
290
|
+
def is_dir(self) -> bool:
|
|
291
|
+
"""Is the stage object a directory?"""
|
|
292
|
+
return self.type == 'directory'
|
|
293
|
+
|
|
294
|
+
def is_file(self) -> bool:
|
|
295
|
+
"""Is the stage object a file?"""
|
|
296
|
+
return self.type != 'directory'
|
|
297
|
+
|
|
298
|
+
def abspath(self) -> str:
|
|
299
|
+
"""Return the full path of the object."""
|
|
300
|
+
return str(self.path)
|
|
301
|
+
|
|
302
|
+
def basename(self) -> str:
|
|
303
|
+
"""Return the basename of the object."""
|
|
304
|
+
return self.name
|
|
305
|
+
|
|
306
|
+
def dirname(self) -> str:
|
|
307
|
+
"""Return the directory name of the object."""
|
|
308
|
+
return re.sub(r'/*$', r'', os.path.dirname(re.sub(r'/*$', r'', self.path))) + '/'
|
|
309
|
+
|
|
310
|
+
def getmtime(self) -> float:
|
|
311
|
+
"""Return the last modified datetime as a UNIX timestamp."""
|
|
312
|
+
if self.last_modified_at is None:
|
|
313
|
+
return 0.0
|
|
314
|
+
return self.last_modified_at.timestamp()
|
|
315
|
+
|
|
316
|
+
def getctime(self) -> float:
|
|
317
|
+
"""Return the creation datetime as a UNIX timestamp."""
|
|
318
|
+
if self.created_at is None:
|
|
319
|
+
return 0.0
|
|
320
|
+
return self.created_at.timestamp()
|
|
321
|
+
|
|
322
|
+
|
|
323
|
+
class StageObjectTextWriter(io.StringIO):
|
|
324
|
+
"""StringIO wrapper for writing to Stage."""
|
|
325
|
+
|
|
326
|
+
def __init__(self, buffer: Optional[str], stage: Stage, stage_path: PathLike):
|
|
327
|
+
self._stage = stage
|
|
328
|
+
self._stage_path = stage_path
|
|
329
|
+
super().__init__(buffer)
|
|
330
|
+
|
|
331
|
+
def close(self) -> None:
|
|
332
|
+
"""Write the content to the stage path."""
|
|
333
|
+
self._stage._upload(self.getvalue(), self._stage_path)
|
|
334
|
+
super().close()
|
|
335
|
+
|
|
336
|
+
|
|
337
|
+
class StageObjectTextReader(io.StringIO):
|
|
338
|
+
"""StringIO wrapper for reading from Stage."""
|
|
339
|
+
|
|
340
|
+
|
|
341
|
+
class StageObjectBytesWriter(io.BytesIO):
|
|
342
|
+
"""BytesIO wrapper for writing to Stage."""
|
|
343
|
+
|
|
344
|
+
def __init__(self, buffer: bytes, stage: Stage, stage_path: PathLike):
|
|
345
|
+
self._stage = stage
|
|
346
|
+
self._stage_path = stage_path
|
|
347
|
+
super().__init__(buffer)
|
|
348
|
+
|
|
349
|
+
def close(self) -> None:
|
|
350
|
+
"""Write the content to the stage path."""
|
|
351
|
+
self._stage._upload(self.getvalue(), self._stage_path)
|
|
352
|
+
super().close()
|
|
353
|
+
|
|
354
|
+
|
|
355
|
+
class StageObjectBytesReader(io.BytesIO):
|
|
356
|
+
"""BytesIO wrapper for reading from Stage."""
|
|
357
|
+
|
|
358
|
+
|
|
359
|
+
class Stage(object):
|
|
360
|
+
"""
|
|
361
|
+
Stage manager.
|
|
362
|
+
|
|
363
|
+
This object is not instantiated directly.
|
|
364
|
+
It is returned by ``WorkspaceGroup.stage``.
|
|
365
|
+
|
|
366
|
+
"""
|
|
367
|
+
|
|
368
|
+
def __init__(self, workspace_group: WorkspaceGroup, manager: WorkspaceManager):
|
|
369
|
+
self._workspace_group = workspace_group
|
|
370
|
+
self._manager = manager
|
|
371
|
+
|
|
372
|
+
def open(
|
|
373
|
+
self,
|
|
374
|
+
stage_path: PathLike,
|
|
375
|
+
mode: str = 'r',
|
|
376
|
+
encoding: Optional[str] = None,
|
|
377
|
+
) -> Union[io.StringIO, io.BytesIO]:
|
|
378
|
+
"""
|
|
379
|
+
Open a Stage path for reading or writing.
|
|
380
|
+
|
|
381
|
+
Parameters
|
|
382
|
+
----------
|
|
383
|
+
stage_path : Path or str
|
|
384
|
+
The stage path to read / write
|
|
385
|
+
mode : str, optional
|
|
386
|
+
The read / write mode. The following modes are supported:
|
|
387
|
+
* 'r' open for reading (default)
|
|
388
|
+
* 'w' open for writing, truncating the file first
|
|
389
|
+
* 'x' create a new file and open it for writing
|
|
390
|
+
The data type can be specified by adding one of the following:
|
|
391
|
+
* 'b' binary mode
|
|
392
|
+
* 't' text mode (default)
|
|
393
|
+
encoding : str, optional
|
|
394
|
+
The string encoding to use for text
|
|
395
|
+
|
|
396
|
+
Returns
|
|
397
|
+
-------
|
|
398
|
+
StageObjectBytesReader - 'rb' or 'b' mode
|
|
399
|
+
StageObjectBytesWriter - 'wb' or 'xb' mode
|
|
400
|
+
StageObjectTextReader - 'r' or 'rt' mode
|
|
401
|
+
StageObjectTextWriter - 'w', 'x', 'wt' or 'xt' mode
|
|
402
|
+
|
|
403
|
+
"""
|
|
404
|
+
if '+' in mode or 'a' in mode:
|
|
405
|
+
raise ValueError('modifying an existing stage file is not supported')
|
|
406
|
+
|
|
407
|
+
if 'w' in mode or 'x' in mode:
|
|
408
|
+
exists = self.exists(stage_path)
|
|
409
|
+
if exists:
|
|
410
|
+
if 'x' in mode:
|
|
411
|
+
raise FileExistsError(f'stage path already exists: {stage_path}')
|
|
412
|
+
self.remove(stage_path)
|
|
413
|
+
if 'b' in mode:
|
|
414
|
+
return StageObjectBytesWriter(b'', self, stage_path)
|
|
415
|
+
return StageObjectTextWriter('', self, stage_path)
|
|
416
|
+
|
|
417
|
+
if 'r' in mode:
|
|
418
|
+
content = self.download_file(stage_path)
|
|
419
|
+
if isinstance(content, bytes):
|
|
420
|
+
if 'b' in mode:
|
|
421
|
+
return StageObjectBytesReader(content)
|
|
422
|
+
encoding = 'utf-8' if encoding is None else encoding
|
|
423
|
+
return StageObjectTextReader(content.decode(encoding))
|
|
424
|
+
|
|
425
|
+
if isinstance(content, str):
|
|
426
|
+
return StageObjectTextReader(content)
|
|
427
|
+
|
|
428
|
+
raise ValueError(f'unrecognized file content type: {type(content)}')
|
|
429
|
+
|
|
430
|
+
raise ValueError(f'must have one of create/read/write mode specified: {mode}')
|
|
431
|
+
|
|
432
|
+
def upload_file(
|
|
433
|
+
self,
|
|
434
|
+
local_path: Union[PathLike, TextIO, BinaryIO],
|
|
435
|
+
stage_path: PathLike,
|
|
436
|
+
*,
|
|
437
|
+
overwrite: bool = False,
|
|
438
|
+
) -> StageObject:
|
|
439
|
+
"""
|
|
440
|
+
Upload a local file.
|
|
441
|
+
|
|
442
|
+
Parameters
|
|
443
|
+
----------
|
|
444
|
+
local_path : Path or str or file-like
|
|
445
|
+
Path to the local file or an open file object
|
|
446
|
+
stage_path : Path or str
|
|
447
|
+
Path to the stage file
|
|
448
|
+
overwrite : bool, optional
|
|
449
|
+
Should the ``stage_path`` be overwritten if it exists already?
|
|
450
|
+
|
|
451
|
+
"""
|
|
452
|
+
if isinstance(local_path, (TextIO, BinaryIO)):
|
|
453
|
+
pass
|
|
454
|
+
elif not os.path.isfile(local_path):
|
|
455
|
+
raise IsADirectoryError(f'local path is not a file: {local_path}')
|
|
456
|
+
|
|
457
|
+
if self.exists(stage_path):
|
|
458
|
+
if not overwrite:
|
|
459
|
+
raise OSError(f'stage path already exists: {stage_path}')
|
|
460
|
+
|
|
461
|
+
self.remove(stage_path)
|
|
462
|
+
|
|
463
|
+
if isinstance(local_path, (TextIO, BinaryIO)):
|
|
464
|
+
return self._upload(local_path, stage_path, overwrite=overwrite)
|
|
465
|
+
return self._upload(open(local_path, 'rb'), stage_path, overwrite=overwrite)
|
|
466
|
+
|
|
467
|
+
def upload_folder(
|
|
468
|
+
self,
|
|
469
|
+
local_path: PathLike,
|
|
470
|
+
stage_path: PathLike,
|
|
471
|
+
*,
|
|
472
|
+
overwrite: bool = False,
|
|
473
|
+
recursive: bool = True,
|
|
474
|
+
include_root: bool = False,
|
|
475
|
+
ignore: Optional[Union[PathLike, List[PathLike]]] = None,
|
|
476
|
+
) -> StageObject:
|
|
477
|
+
"""
|
|
478
|
+
Upload a folder recursively.
|
|
479
|
+
|
|
480
|
+
Only the contents of the folder are uploaded. To include the
|
|
481
|
+
folder name itself in the target path use ``include_root=True``.
|
|
482
|
+
|
|
483
|
+
Parameters
|
|
484
|
+
----------
|
|
485
|
+
local_path : Path or str
|
|
486
|
+
Local directory to upload
|
|
487
|
+
stage_path : Path or str
|
|
488
|
+
Path of stage folder to upload to
|
|
489
|
+
overwrite : bool, optional
|
|
490
|
+
If a file already exists, should it be overwritten?
|
|
491
|
+
recursive : bool, optional
|
|
492
|
+
Should nested folders be uploaded?
|
|
493
|
+
include_root : bool, optional
|
|
494
|
+
Should the local root folder itself be uploaded as the top folder?
|
|
495
|
+
ignore : Path or str or List[Path] or List[str], optional
|
|
496
|
+
Glob patterns of files to ignore, for example, '**/*.pyc` will
|
|
497
|
+
ignore all '*.pyc' files in the directory tree
|
|
498
|
+
|
|
499
|
+
"""
|
|
500
|
+
if not os.path.isdir(local_path):
|
|
501
|
+
raise NotADirectoryError(f'local path is not a directory: {local_path}')
|
|
502
|
+
if self.exists(stage_path) and not self.is_dir(stage_path):
|
|
503
|
+
raise NotADirectoryError(f'stage path is not a directory: {stage_path}')
|
|
504
|
+
|
|
505
|
+
ignore_files = set()
|
|
506
|
+
if ignore:
|
|
507
|
+
if isinstance(ignore, list):
|
|
508
|
+
for item in ignore:
|
|
509
|
+
ignore_files.update(glob.glob(str(item), recursive=recursive))
|
|
510
|
+
else:
|
|
511
|
+
ignore_files.update(glob.glob(str(ignore), recursive=recursive))
|
|
512
|
+
|
|
513
|
+
parent_dir = os.path.basename(os.getcwd())
|
|
514
|
+
|
|
515
|
+
files = glob.glob(os.path.join(local_path, '**'), recursive=recursive)
|
|
516
|
+
|
|
517
|
+
for src in files:
|
|
518
|
+
if ignore_files and src in ignore_files:
|
|
519
|
+
continue
|
|
520
|
+
target = os.path.join(parent_dir, src) if include_root else src
|
|
521
|
+
self.upload_file(src, target, overwrite=overwrite)
|
|
522
|
+
|
|
523
|
+
return self.info(stage_path)
|
|
524
|
+
|
|
525
|
+
def _upload(
|
|
526
|
+
self,
|
|
527
|
+
content: Union[str, bytes, TextIO, BinaryIO],
|
|
528
|
+
stage_path: PathLike,
|
|
529
|
+
*,
|
|
530
|
+
overwrite: bool = False,
|
|
531
|
+
) -> StageObject:
|
|
532
|
+
"""
|
|
533
|
+
Upload content to a stage file.
|
|
534
|
+
|
|
535
|
+
Parameters
|
|
536
|
+
----------
|
|
537
|
+
content : str or bytes or file-like
|
|
538
|
+
Content to upload to stage
|
|
539
|
+
stage_path : Path or str
|
|
540
|
+
Path to the stage file
|
|
541
|
+
overwrite : bool, optional
|
|
542
|
+
Should the ``stage_path`` be overwritten if it exists already?
|
|
543
|
+
|
|
544
|
+
"""
|
|
545
|
+
if self.exists(stage_path):
|
|
546
|
+
if not overwrite:
|
|
547
|
+
raise OSError(f'stage path already exists: {stage_path}')
|
|
548
|
+
self.remove(stage_path)
|
|
549
|
+
|
|
550
|
+
self._manager._put(
|
|
551
|
+
f'stage/{self._workspace_group.id}/fs/{stage_path}',
|
|
552
|
+
files={'file': content},
|
|
553
|
+
headers={'Content-Type': None},
|
|
554
|
+
)
|
|
555
|
+
|
|
556
|
+
return self.info(stage_path)
|
|
557
|
+
|
|
558
|
+
def mkdir(self, stage_path: PathLike, overwrite: bool = False) -> StageObject:
|
|
559
|
+
"""
|
|
560
|
+
Make a directory in the stage.
|
|
561
|
+
|
|
562
|
+
Parameters
|
|
563
|
+
----------
|
|
564
|
+
stage_path : Path or str
|
|
565
|
+
Path of the folder to create
|
|
566
|
+
overwrite : bool, optional
|
|
567
|
+
Should the stage path be overwritten if it exists already?
|
|
568
|
+
|
|
569
|
+
Returns
|
|
570
|
+
-------
|
|
571
|
+
StageObject
|
|
572
|
+
|
|
573
|
+
"""
|
|
574
|
+
stage_path = re.sub(r'/*$', r'', str(stage_path)) + '/'
|
|
575
|
+
|
|
576
|
+
if self.exists(stage_path):
|
|
577
|
+
if not overwrite:
|
|
578
|
+
return self.info(stage_path)
|
|
579
|
+
|
|
580
|
+
self.remove(stage_path)
|
|
581
|
+
|
|
582
|
+
self._manager._put(
|
|
583
|
+
f'stage/{self._workspace_group.id}/fs/{stage_path}?isFile=false',
|
|
584
|
+
)
|
|
585
|
+
|
|
586
|
+
return self.info(stage_path)
|
|
587
|
+
|
|
588
|
+
mkdirs = mkdir
|
|
589
|
+
|
|
590
|
+
def rename(
|
|
591
|
+
self,
|
|
592
|
+
old_path: PathLike,
|
|
593
|
+
new_path: PathLike,
|
|
594
|
+
*,
|
|
595
|
+
overwrite: bool = False,
|
|
596
|
+
) -> StageObject:
|
|
597
|
+
"""
|
|
598
|
+
Move the stage file to a new location.
|
|
599
|
+
|
|
600
|
+
Paraemeters
|
|
601
|
+
-----------
|
|
602
|
+
old_path : Path or str
|
|
603
|
+
Original location of the path
|
|
604
|
+
new_path : Path or str
|
|
605
|
+
New location of the path
|
|
606
|
+
overwrite : bool, optional
|
|
607
|
+
Should the ``new_path`` be overwritten if it exists already?
|
|
608
|
+
|
|
609
|
+
"""
|
|
610
|
+
if not self.exists(old_path):
|
|
611
|
+
raise OSError(f'stage path does not exist: {old_path}')
|
|
612
|
+
|
|
613
|
+
if self.exists(new_path):
|
|
614
|
+
if not overwrite:
|
|
615
|
+
raise OSError(f'stage path already exists: {new_path}')
|
|
616
|
+
|
|
617
|
+
if str(old_path).endswith('/') and not str(new_path).endswith('/'):
|
|
618
|
+
raise OSError('original and new paths are not the same type')
|
|
619
|
+
|
|
620
|
+
if str(new_path).endswith('/'):
|
|
621
|
+
self.removedirs(new_path)
|
|
622
|
+
else:
|
|
623
|
+
self.remove(new_path)
|
|
624
|
+
|
|
625
|
+
self._manager._patch(
|
|
626
|
+
f'stage/{self._workspace_group.id}/fs/{old_path}',
|
|
627
|
+
json=dict(newPath=new_path),
|
|
628
|
+
)
|
|
629
|
+
|
|
630
|
+
return self.info(new_path)
|
|
631
|
+
|
|
632
|
+
def info(self, stage_path: PathLike) -> StageObject:
|
|
633
|
+
"""
|
|
634
|
+
Return information about a stage location.
|
|
635
|
+
|
|
636
|
+
Parameters
|
|
637
|
+
----------
|
|
638
|
+
stage_path : Path or str
|
|
639
|
+
Path to the stage location
|
|
640
|
+
|
|
641
|
+
Returns
|
|
642
|
+
-------
|
|
643
|
+
StageObject
|
|
644
|
+
|
|
645
|
+
"""
|
|
646
|
+
res = self._manager._get(
|
|
647
|
+
re.sub(r'/+$', r'/', f'stage/{self._workspace_group.id}/fs/{stage_path}'),
|
|
648
|
+
params=dict(metadata=1),
|
|
649
|
+
).json()
|
|
650
|
+
|
|
651
|
+
return StageObject.from_dict(res, self)
|
|
652
|
+
|
|
653
|
+
def exists(self, stage_path: PathLike) -> bool:
|
|
654
|
+
"""
|
|
655
|
+
Does the given stage path exist?
|
|
656
|
+
|
|
657
|
+
Parameters
|
|
658
|
+
----------
|
|
659
|
+
stage_path : Path or str
|
|
660
|
+
Path to stage object
|
|
661
|
+
|
|
662
|
+
Returns
|
|
663
|
+
-------
|
|
664
|
+
bool
|
|
665
|
+
|
|
666
|
+
"""
|
|
667
|
+
try:
|
|
668
|
+
self.info(stage_path)
|
|
669
|
+
return True
|
|
670
|
+
except ManagementError as exc:
|
|
671
|
+
if exc.errno == 404:
|
|
672
|
+
return False
|
|
673
|
+
raise
|
|
674
|
+
|
|
675
|
+
def is_dir(self, stage_path: PathLike) -> bool:
|
|
676
|
+
"""
|
|
677
|
+
Is the given stage path a directory?
|
|
678
|
+
|
|
679
|
+
Parameters
|
|
680
|
+
----------
|
|
681
|
+
stage_path : Path or str
|
|
682
|
+
Path to stage object
|
|
683
|
+
|
|
684
|
+
Returns
|
|
685
|
+
-------
|
|
686
|
+
bool
|
|
687
|
+
|
|
688
|
+
"""
|
|
689
|
+
try:
|
|
690
|
+
return self.info(stage_path).type == 'directory'
|
|
691
|
+
except ManagementError as exc:
|
|
692
|
+
if exc.errno == 404:
|
|
693
|
+
return False
|
|
694
|
+
raise
|
|
695
|
+
|
|
696
|
+
def is_file(self, stage_path: PathLike) -> bool:
|
|
697
|
+
"""
|
|
698
|
+
Is the given stage path a file?
|
|
699
|
+
|
|
700
|
+
Parameters
|
|
701
|
+
----------
|
|
702
|
+
stage_path : Path or str
|
|
703
|
+
Path to stage object
|
|
704
|
+
|
|
705
|
+
Returns
|
|
706
|
+
-------
|
|
707
|
+
bool
|
|
708
|
+
|
|
709
|
+
"""
|
|
710
|
+
try:
|
|
711
|
+
return self.info(stage_path).type != 'directory'
|
|
712
|
+
except ManagementError as exc:
|
|
713
|
+
if exc.errno == 404:
|
|
714
|
+
return False
|
|
715
|
+
raise
|
|
716
|
+
|
|
717
|
+
def _listdir(self, stage_path: PathLike, *, recursive: bool = False) -> List[str]:
|
|
718
|
+
"""
|
|
719
|
+
Return the names of files in a directory.
|
|
720
|
+
|
|
721
|
+
Parameters
|
|
722
|
+
----------
|
|
723
|
+
stage_path : Path or str
|
|
724
|
+
Path to the folder in Stage
|
|
725
|
+
recursive : bool, optional
|
|
726
|
+
Should folders be listed recursively?
|
|
727
|
+
|
|
728
|
+
"""
|
|
729
|
+
res = self._manager._get(
|
|
730
|
+
f'stage/{self._workspace_group.id}/fs/{stage_path}',
|
|
731
|
+
).json()
|
|
732
|
+
if recursive:
|
|
733
|
+
out = []
|
|
734
|
+
for item in res['content'] or []:
|
|
735
|
+
out.append(item['path'])
|
|
736
|
+
if item['type'] == 'directory':
|
|
737
|
+
out.extend(self._listdir(item['path'], recursive=recursive))
|
|
738
|
+
return out
|
|
739
|
+
return [x['path'] for x in res['content'] or []]
|
|
740
|
+
|
|
741
|
+
def listdir(
|
|
742
|
+
self,
|
|
743
|
+
stage_path: PathLike = '/',
|
|
744
|
+
*,
|
|
745
|
+
recursive: bool = False,
|
|
746
|
+
) -> List[str]:
|
|
747
|
+
"""
|
|
748
|
+
List the files / folders at the given path.
|
|
749
|
+
|
|
750
|
+
Parameters
|
|
751
|
+
----------
|
|
752
|
+
stage_path : Path or str, optional
|
|
753
|
+
Path to the stage location
|
|
754
|
+
|
|
755
|
+
Returns
|
|
756
|
+
-------
|
|
757
|
+
List[str]
|
|
758
|
+
|
|
759
|
+
"""
|
|
760
|
+
stage_path = re.sub(r'^(\./|/)+', r'', str(stage_path))
|
|
761
|
+
stage_path = re.sub(r'/+$', r'', stage_path) + '/'
|
|
762
|
+
|
|
763
|
+
if self.is_dir(stage_path):
|
|
764
|
+
out = self._listdir(stage_path, recursive=recursive)
|
|
765
|
+
if stage_path != '/':
|
|
766
|
+
stage_path_n = len(stage_path.split('/')) - 1
|
|
767
|
+
out = ['/'.join(x.split('/')[stage_path_n:]) for x in out]
|
|
768
|
+
return out
|
|
769
|
+
|
|
770
|
+
raise NotADirectoryError(f'stage path is not a directory: {stage_path}')
|
|
771
|
+
|
|
772
|
+
def download_file(
|
|
773
|
+
self,
|
|
774
|
+
stage_path: PathLike,
|
|
775
|
+
local_path: Optional[PathLike] = None,
|
|
776
|
+
*,
|
|
777
|
+
overwrite: bool = False,
|
|
778
|
+
encoding: Optional[str] = None,
|
|
779
|
+
) -> Optional[Union[bytes, str]]:
|
|
780
|
+
"""
|
|
781
|
+
Download the content of a stage path.
|
|
782
|
+
|
|
783
|
+
Parameters
|
|
784
|
+
----------
|
|
785
|
+
stage_path : Path or str
|
|
786
|
+
Path to the stage file
|
|
787
|
+
local_path : Path or str
|
|
788
|
+
Path to local file target location
|
|
789
|
+
overwrite : bool, optional
|
|
790
|
+
Should an existing file be overwritten if it exists?
|
|
791
|
+
encoding : str, optional
|
|
792
|
+
Encoding used to convert the resulting data
|
|
793
|
+
|
|
794
|
+
Returns
|
|
795
|
+
-------
|
|
796
|
+
bytes or str - ``local_path`` is None
|
|
797
|
+
None - ``local_path`` is a Path or str
|
|
798
|
+
|
|
799
|
+
"""
|
|
800
|
+
if local_path is not None and not overwrite and os.path.exists(local_path):
|
|
801
|
+
raise OSError('target file already exists; use overwrite=True to replace')
|
|
802
|
+
if self.is_dir(stage_path):
|
|
803
|
+
raise IsADirectoryError(f'stage path is a directory: {stage_path}')
|
|
804
|
+
|
|
805
|
+
out = self._manager._get(
|
|
806
|
+
f'stage/{self._workspace_group.id}/fs/{stage_path}',
|
|
807
|
+
).content
|
|
808
|
+
|
|
809
|
+
if local_path is not None:
|
|
810
|
+
with open(local_path, 'wb') as outfile:
|
|
811
|
+
outfile.write(out)
|
|
812
|
+
return None
|
|
813
|
+
|
|
814
|
+
if encoding:
|
|
815
|
+
return out.decode(encoding)
|
|
816
|
+
|
|
817
|
+
return out
|
|
818
|
+
|
|
819
|
+
def download_folder(
|
|
820
|
+
self,
|
|
821
|
+
stage_path: PathLike,
|
|
822
|
+
local_path: PathLike = '.',
|
|
823
|
+
*,
|
|
824
|
+
overwrite: bool = False,
|
|
825
|
+
) -> None:
|
|
826
|
+
"""
|
|
827
|
+
Download a Stage folder to a local directory.
|
|
828
|
+
|
|
829
|
+
Parameters
|
|
830
|
+
----------
|
|
831
|
+
stage_path : Path or str
|
|
832
|
+
Path to the stage file
|
|
833
|
+
local_path : Path or str
|
|
834
|
+
Path to local directory target location
|
|
835
|
+
overwrite : bool, optional
|
|
836
|
+
Should an existing directory / files be overwritten if they exist?
|
|
837
|
+
|
|
838
|
+
"""
|
|
839
|
+
if local_path is not None and not overwrite and os.path.exists(local_path):
|
|
840
|
+
raise OSError(
|
|
841
|
+
'target directory already exists; '
|
|
842
|
+
'use overwrite=True to replace',
|
|
843
|
+
)
|
|
844
|
+
if not self.is_dir(stage_path):
|
|
845
|
+
raise NotADirectoryError(f'stage path is not a directory: {stage_path}')
|
|
846
|
+
|
|
847
|
+
for f in self.listdir(stage_path, recursive=True):
|
|
848
|
+
if self.is_dir(f):
|
|
849
|
+
continue
|
|
850
|
+
target = os.path.normpath(os.path.join(local_path, f))
|
|
851
|
+
os.makedirs(os.path.dirname(target), exist_ok=True)
|
|
852
|
+
self.download_file(f, target, overwrite=overwrite)
|
|
853
|
+
|
|
854
|
+
def remove(self, stage_path: PathLike) -> None:
|
|
855
|
+
"""
|
|
856
|
+
Delete a stage location.
|
|
857
|
+
|
|
858
|
+
Parameters
|
|
859
|
+
----------
|
|
860
|
+
stage_path : Path or str
|
|
861
|
+
Path to the stage location
|
|
862
|
+
|
|
863
|
+
"""
|
|
864
|
+
if self.is_dir(stage_path):
|
|
865
|
+
raise IsADirectoryError(
|
|
866
|
+
'stage path is a directory, '
|
|
867
|
+
f'use rmdir or removedirs: {stage_path}',
|
|
868
|
+
)
|
|
869
|
+
|
|
870
|
+
self._manager._delete(f'stage/{self._workspace_group.id}/fs/{stage_path}')
|
|
871
|
+
|
|
872
|
+
def removedirs(self, stage_path: PathLike) -> None:
|
|
873
|
+
"""
|
|
874
|
+
Delete a stage folder recursively.
|
|
875
|
+
|
|
876
|
+
Parameters
|
|
877
|
+
----------
|
|
878
|
+
stage_path : Path or str
|
|
879
|
+
Path to the stage location
|
|
880
|
+
|
|
881
|
+
"""
|
|
882
|
+
stage_path = re.sub(r'/*$', r'', str(stage_path)) + '/'
|
|
883
|
+
self._manager._delete(f'stage/{self._workspace_group.id}/fs/{stage_path}')
|
|
884
|
+
|
|
885
|
+
def rmdir(self, stage_path: PathLike) -> None:
|
|
886
|
+
"""
|
|
887
|
+
Delete a stage folder.
|
|
888
|
+
|
|
889
|
+
Parameters
|
|
890
|
+
----------
|
|
891
|
+
stage_path : Path or str
|
|
892
|
+
Path to the stage location
|
|
893
|
+
|
|
894
|
+
"""
|
|
895
|
+
stage_path = re.sub(r'/*$', r'', str(stage_path)) + '/'
|
|
896
|
+
|
|
897
|
+
if self.listdir(stage_path):
|
|
898
|
+
raise OSError(f'stage folder is not empty, use removedirs: {stage_path}')
|
|
899
|
+
|
|
900
|
+
self._manager._delete(f'stage/{self._workspace_group.id}/fs/{stage_path}')
|
|
901
|
+
|
|
902
|
+
def __str__(self) -> str:
|
|
903
|
+
"""Return string representation."""
|
|
904
|
+
return vars_to_str(self)
|
|
905
|
+
|
|
906
|
+
def __repr__(self) -> str:
|
|
907
|
+
"""Return string representation."""
|
|
908
|
+
return str(self)
|
|
909
|
+
|
|
910
|
+
|
|
18
911
|
class Workspace(object):
|
|
19
912
|
"""
|
|
20
913
|
SingleStoreDB workspace definition.
|
|
@@ -34,12 +927,21 @@ class Workspace(object):
|
|
|
34
927
|
"""
|
|
35
928
|
|
|
36
929
|
def __init__(
|
|
37
|
-
self,
|
|
930
|
+
self,
|
|
931
|
+
name: str,
|
|
932
|
+
workspace_id: str,
|
|
38
933
|
workspace_group: Union[str, 'WorkspaceGroup'],
|
|
39
|
-
size: str,
|
|
934
|
+
size: str,
|
|
935
|
+
state: str,
|
|
40
936
|
created_at: Union[str, datetime.datetime],
|
|
41
937
|
terminated_at: Optional[Union[str, datetime.datetime]] = None,
|
|
42
938
|
endpoint: Optional[str] = None,
|
|
939
|
+
auto_suspend: Optional[Dict[str, Any]] = None,
|
|
940
|
+
cache_config: Optional[int] = None,
|
|
941
|
+
deployment_type: Optional[str] = None,
|
|
942
|
+
resume_attachments: Optional[Dict[str, Any]] = None,
|
|
943
|
+
scaling_progress: Optional[int] = None,
|
|
944
|
+
last_resumed_at: Optional[str] = None,
|
|
43
945
|
):
|
|
44
946
|
#: Name of the workspace
|
|
45
947
|
self.name = name
|
|
@@ -69,6 +971,24 @@ class Workspace(object):
|
|
|
69
971
|
#: Hostname (or IP address) of the workspace database server
|
|
70
972
|
self.endpoint = endpoint
|
|
71
973
|
|
|
974
|
+
#: Current auto-suspend settings
|
|
975
|
+
self.auto_suspend = camel_to_snake_dict(auto_suspend)
|
|
976
|
+
|
|
977
|
+
#: Multiplier for the persistent cache
|
|
978
|
+
self.cache_config = cache_config
|
|
979
|
+
|
|
980
|
+
#: Deployment type of the workspace
|
|
981
|
+
self.deployment_type = deployment_type
|
|
982
|
+
|
|
983
|
+
#: Database attachments
|
|
984
|
+
self.resume_attachments = camel_to_snake_dict(resume_attachments)
|
|
985
|
+
|
|
986
|
+
#: Current progress percentage for scaling the workspace
|
|
987
|
+
self.scaling_progress = scaling_progress
|
|
988
|
+
|
|
989
|
+
#: Timestamp when workspace was last resumed
|
|
990
|
+
self.last_resumed_at = to_datetime(last_resumed_at)
|
|
991
|
+
|
|
72
992
|
self._manager: Optional[WorkspaceManager] = None
|
|
73
993
|
|
|
74
994
|
def __str__(self) -> str:
|
|
@@ -97,16 +1017,65 @@ class Workspace(object):
|
|
|
97
1017
|
|
|
98
1018
|
"""
|
|
99
1019
|
out = cls(
|
|
100
|
-
name=obj['name'],
|
|
1020
|
+
name=obj['name'],
|
|
1021
|
+
workspace_id=obj['workspaceID'],
|
|
101
1022
|
workspace_group=obj['workspaceGroupID'],
|
|
102
|
-
size=obj.get('size', 'Unknown'),
|
|
103
|
-
|
|
1023
|
+
size=obj.get('size', 'Unknown'),
|
|
1024
|
+
state=obj['state'],
|
|
1025
|
+
created_at=obj['createdAt'],
|
|
1026
|
+
terminated_at=obj.get('terminatedAt'),
|
|
104
1027
|
endpoint=obj.get('endpoint'),
|
|
1028
|
+
auto_suspend=obj.get('autoSuspend'),
|
|
1029
|
+
cache_config=obj.get('cacheConfig'),
|
|
1030
|
+
deployment_type=obj.get('deploymentType'),
|
|
1031
|
+
last_resumed_at=obj.get('lastResumedAt'),
|
|
1032
|
+
resume_attachments=obj.get('resumeAttachments'),
|
|
1033
|
+
scaling_progress=obj.get('scalingProgress'),
|
|
105
1034
|
)
|
|
106
1035
|
out._manager = manager
|
|
107
1036
|
return out
|
|
108
1037
|
|
|
109
|
-
def
|
|
1038
|
+
def update(
|
|
1039
|
+
self,
|
|
1040
|
+
auto_suspend: Optional[Dict[str, Any]] = None,
|
|
1041
|
+
cache_config: Optional[int] = None,
|
|
1042
|
+
deployment_type: Optional[str] = None,
|
|
1043
|
+
size: Optional[str] = None,
|
|
1044
|
+
) -> None:
|
|
1045
|
+
"""
|
|
1046
|
+
Update the workspace definition.
|
|
1047
|
+
|
|
1048
|
+
Parameters
|
|
1049
|
+
----------
|
|
1050
|
+
auto_suspend : Dict[str, Any], optional
|
|
1051
|
+
Auto-suspend mode for the workspace: IDLE, SCHEDULED, DISABLED
|
|
1052
|
+
cache_config : int, optional
|
|
1053
|
+
Specifies the multiplier for the persistent cache associated
|
|
1054
|
+
with the workspace. If specified, it enables the cache configuration
|
|
1055
|
+
multiplier. It can have one of the following values: 1, 2, or 4.
|
|
1056
|
+
deployment_type : str, optional
|
|
1057
|
+
The deployment type that will be applied to all the workspaces
|
|
1058
|
+
within the group
|
|
1059
|
+
size : str, optional
|
|
1060
|
+
Size of the workspace (in workspace size notation), such as "S-1".
|
|
1061
|
+
|
|
1062
|
+
"""
|
|
1063
|
+
if self._manager is None:
|
|
1064
|
+
raise ManagementError(
|
|
1065
|
+
msg='No workspace manager is associated with this object.',
|
|
1066
|
+
)
|
|
1067
|
+
data = {
|
|
1068
|
+
k: v for k, v in dict(
|
|
1069
|
+
autoSuspend=snake_to_camel_dict(auto_suspend),
|
|
1070
|
+
cacheConfig=cache_config,
|
|
1071
|
+
deploymentType=deployment_type,
|
|
1072
|
+
size=size,
|
|
1073
|
+
).items() if v is not None
|
|
1074
|
+
}
|
|
1075
|
+
self._manager._patch(f'workspaces/{self.id}', json=data)
|
|
1076
|
+
self.refresh()
|
|
1077
|
+
|
|
1078
|
+
def refresh(self) -> Workspace:
|
|
110
1079
|
"""Update the object to the current state."""
|
|
111
1080
|
if self._manager is None:
|
|
112
1081
|
raise ManagementError(
|
|
@@ -114,7 +1083,10 @@ class Workspace(object):
|
|
|
114
1083
|
)
|
|
115
1084
|
new_obj = self._manager.get_workspace(self.id)
|
|
116
1085
|
for name, value in vars(new_obj).items():
|
|
117
|
-
|
|
1086
|
+
if isinstance(value, Mapping):
|
|
1087
|
+
setattr(self, name, snake_to_camel_dict(value))
|
|
1088
|
+
else:
|
|
1089
|
+
setattr(self, name, value)
|
|
118
1090
|
return self
|
|
119
1091
|
|
|
120
1092
|
def terminate(
|
|
@@ -122,6 +1094,7 @@ class Workspace(object):
|
|
|
122
1094
|
wait_on_terminated: bool = False,
|
|
123
1095
|
wait_interval: int = 10,
|
|
124
1096
|
wait_timeout: int = 600,
|
|
1097
|
+
force: bool = False,
|
|
125
1098
|
) -> None:
|
|
126
1099
|
"""
|
|
127
1100
|
Terminate the workspace.
|
|
@@ -134,6 +1107,8 @@ class Workspace(object):
|
|
|
134
1107
|
Number of seconds between each server check
|
|
135
1108
|
wait_timeout : int, optional
|
|
136
1109
|
Total number of seconds to check server before giving up
|
|
1110
|
+
force : bool, optional
|
|
1111
|
+
Should the workspace group be terminated even if it has workspaces?
|
|
137
1112
|
|
|
138
1113
|
Raises
|
|
139
1114
|
------
|
|
@@ -145,7 +1120,8 @@ class Workspace(object):
|
|
|
145
1120
|
raise ManagementError(
|
|
146
1121
|
msg='No workspace manager is associated with this object.',
|
|
147
1122
|
)
|
|
148
|
-
|
|
1123
|
+
force_str = 'true' if force else 'false'
|
|
1124
|
+
self._manager._delete(f'workspaces/{self.id}?force={force_str}')
|
|
149
1125
|
if wait_on_terminated:
|
|
150
1126
|
self._manager._wait_on_state(
|
|
151
1127
|
self._manager.get_workspace(self.id),
|
|
@@ -175,6 +1151,84 @@ class Workspace(object):
|
|
|
175
1151
|
kwargs['host'] = self.endpoint
|
|
176
1152
|
return connection.connect(**kwargs)
|
|
177
1153
|
|
|
1154
|
+
def suspend(
|
|
1155
|
+
self,
|
|
1156
|
+
wait_on_suspended: bool = False,
|
|
1157
|
+
wait_interval: int = 20,
|
|
1158
|
+
wait_timeout: int = 600,
|
|
1159
|
+
) -> None:
|
|
1160
|
+
"""
|
|
1161
|
+
Suspend the workspace.
|
|
1162
|
+
|
|
1163
|
+
Parameters
|
|
1164
|
+
----------
|
|
1165
|
+
wait_on_suspended : bool, optional
|
|
1166
|
+
Wait for the workspace to go into 'Suspended' mode before returning
|
|
1167
|
+
wait_interval : int, optional
|
|
1168
|
+
Number of seconds between each server check
|
|
1169
|
+
wait_timeout : int, optional
|
|
1170
|
+
Total number of seconds to check server before giving up
|
|
1171
|
+
|
|
1172
|
+
Raises
|
|
1173
|
+
------
|
|
1174
|
+
ManagementError
|
|
1175
|
+
If timeout is reached
|
|
1176
|
+
|
|
1177
|
+
"""
|
|
1178
|
+
if self._manager is None:
|
|
1179
|
+
raise ManagementError(
|
|
1180
|
+
msg='No workspace manager is associated with this object.',
|
|
1181
|
+
)
|
|
1182
|
+
self._manager._post(f'workspaces/{self.id}/suspend')
|
|
1183
|
+
if wait_on_suspended:
|
|
1184
|
+
self._manager._wait_on_state(
|
|
1185
|
+
self._manager.get_workspace(self.id),
|
|
1186
|
+
'Suspended', interval=wait_interval, timeout=wait_timeout,
|
|
1187
|
+
)
|
|
1188
|
+
self.refresh()
|
|
1189
|
+
|
|
1190
|
+
def resume(
|
|
1191
|
+
self,
|
|
1192
|
+
disable_auto_suspend: bool = False,
|
|
1193
|
+
wait_on_resumed: bool = False,
|
|
1194
|
+
wait_interval: int = 20,
|
|
1195
|
+
wait_timeout: int = 600,
|
|
1196
|
+
) -> None:
|
|
1197
|
+
"""
|
|
1198
|
+
Resume the workspace.
|
|
1199
|
+
|
|
1200
|
+
Parameters
|
|
1201
|
+
----------
|
|
1202
|
+
disable_auto_suspend : bool, optional
|
|
1203
|
+
Should auto-suspend be disabled?
|
|
1204
|
+
wait_on_resumed : bool, optional
|
|
1205
|
+
Wait for the workspace to go into 'Resumed' or 'Active' mode before returning
|
|
1206
|
+
wait_interval : int, optional
|
|
1207
|
+
Number of seconds between each server check
|
|
1208
|
+
wait_timeout : int, optional
|
|
1209
|
+
Total number of seconds to check server before giving up
|
|
1210
|
+
|
|
1211
|
+
Raises
|
|
1212
|
+
------
|
|
1213
|
+
ManagementError
|
|
1214
|
+
If timeout is reached
|
|
1215
|
+
|
|
1216
|
+
"""
|
|
1217
|
+
if self._manager is None:
|
|
1218
|
+
raise ManagementError(
|
|
1219
|
+
msg='No workspace manager is associated with this object.',
|
|
1220
|
+
)
|
|
1221
|
+
self._manager._post(
|
|
1222
|
+
f'workspaces/{self.id}/resume',
|
|
1223
|
+
json=dict(disableAutoSuspend=disable_auto_suspend),
|
|
1224
|
+
)
|
|
1225
|
+
if wait_on_resumed:
|
|
1226
|
+
self._manager._wait_on_state(
|
|
1227
|
+
self._manager.get_workspace(self.id),
|
|
1228
|
+
['Resumed', 'Active'], interval=wait_interval, timeout=wait_timeout,
|
|
1229
|
+
)
|
|
1230
|
+
self.refresh()
|
|
1231
|
+
|
|
178
1232
|
|
|
179
1233
|
class WorkspaceGroup(object):
|
|
180
1234
|
"""
|
|
@@ -197,9 +1251,10 @@ class WorkspaceGroup(object):
|
|
|
197
1251
|
def __init__(
|
|
198
1252
|
self, name: str, id: str,
|
|
199
1253
|
created_at: Union[str, datetime.datetime],
|
|
200
|
-
region: Region,
|
|
1254
|
+
region: Optional[Region],
|
|
201
1255
|
firewall_ranges: List[str],
|
|
202
1256
|
terminated_at: Optional[Union[str, datetime.datetime]],
|
|
1257
|
+
allow_all_traffic: Optional[bool],
|
|
203
1258
|
):
|
|
204
1259
|
#: Name of the workspace group
|
|
205
1260
|
self.name = name
|
|
@@ -210,7 +1265,7 @@ class WorkspaceGroup(object):
|
|
|
210
1265
|
#: Timestamp of when the workspace group was created
|
|
211
1266
|
self.created_at = to_datetime(created_at)
|
|
212
1267
|
|
|
213
|
-
#: Region of the
|
|
1268
|
+
#: Region of the workspace group (see :class:`Region`)
|
|
214
1269
|
self.region = region
|
|
215
1270
|
|
|
216
1271
|
#: List of allowed incoming IP addresses / ranges
|
|
@@ -219,6 +1274,9 @@ class WorkspaceGroup(object):
|
|
|
219
1274
|
#: Timestamp of when the workspace group was terminated
|
|
220
1275
|
self.terminated_at = to_datetime(terminated_at)
|
|
221
1276
|
|
|
1277
|
+
#: Should all traffic be allowed?
|
|
1278
|
+
self.allow_all_traffic = allow_all_traffic
|
|
1279
|
+
|
|
222
1280
|
self._manager: Optional[WorkspaceManager] = None
|
|
223
1281
|
|
|
224
1282
|
def __str__(self) -> str:
|
|
@@ -248,43 +1306,88 @@ class WorkspaceGroup(object):
|
|
|
248
1306
|
:class:`WorkspaceGroup`
|
|
249
1307
|
|
|
250
1308
|
"""
|
|
1309
|
+
try:
|
|
1310
|
+
region = [x for x in manager.regions if x.id == obj['regionID']][0]
|
|
1311
|
+
except IndexError:
|
|
1312
|
+
region = Region(obj.get('regionID', '<unknown>'), '<unknown>', '<unknown>')
|
|
251
1313
|
out = cls(
|
|
252
|
-
name=obj['name'],
|
|
1314
|
+
name=obj['name'],
|
|
1315
|
+
id=obj['workspaceGroupID'],
|
|
253
1316
|
created_at=obj['createdAt'],
|
|
254
|
-
region=
|
|
1317
|
+
region=region,
|
|
255
1318
|
firewall_ranges=obj.get('firewallRanges', []),
|
|
256
1319
|
terminated_at=obj.get('terminatedAt'),
|
|
1320
|
+
allow_all_traffic=obj.get('allowAllTraffic'),
|
|
257
1321
|
)
|
|
258
1322
|
out._manager = manager
|
|
259
1323
|
return out
|
|
260
1324
|
|
|
1325
|
+
@property
|
|
1326
|
+
def organization(self) -> Organization:
|
|
1327
|
+
if self._manager is None:
|
|
1328
|
+
raise ManagementError(
|
|
1329
|
+
msg='No workspace manager is associated with this object.',
|
|
1330
|
+
)
|
|
1331
|
+
return self._manager.organization
|
|
1332
|
+
|
|
1333
|
+
@property
|
|
1334
|
+
def stage(self) -> Stage:
|
|
1335
|
+
"""Stage manager."""
|
|
1336
|
+
if self._manager is None:
|
|
1337
|
+
raise ManagementError(
|
|
1338
|
+
msg='No workspace manager is associated with this object.',
|
|
1339
|
+
)
|
|
1340
|
+
return Stage(self, self._manager)
|
|
1341
|
+
|
|
1342
|
+
stages = stage
|
|
1343
|
+
|
|
261
1344
|
def refresh(self) -> 'WorkspaceGroup':
|
|
262
|
-
"""Update
|
|
1345
|
+
"""Update the object to the current state."""
|
|
263
1346
|
if self._manager is None:
|
|
264
1347
|
raise ManagementError(
|
|
265
1348
|
msg='No workspace manager is associated with this object.',
|
|
266
1349
|
)
|
|
267
1350
|
new_obj = self._manager.get_workspace_group(self.id)
|
|
268
1351
|
for name, value in vars(new_obj).items():
|
|
269
|
-
|
|
1352
|
+
if isinstance(value, Mapping):
|
|
1353
|
+
setattr(self, name, camel_to_snake_dict(value))
|
|
1354
|
+
else:
|
|
1355
|
+
setattr(self, name, value)
|
|
270
1356
|
return self
|
|
271
1357
|
|
|
272
1358
|
def update(
|
|
273
|
-
self,
|
|
274
|
-
|
|
1359
|
+
self,
|
|
1360
|
+
name: Optional[str] = None,
|
|
275
1361
|
firewall_ranges: Optional[List[str]] = None,
|
|
1362
|
+
admin_password: Optional[str] = None,
|
|
1363
|
+
expires_at: Optional[str] = None,
|
|
1364
|
+
allow_all_traffic: Optional[bool] = None,
|
|
1365
|
+
update_window: Optional[Dict[str, int]] = None,
|
|
276
1366
|
) -> None:
|
|
277
1367
|
"""
|
|
278
|
-
Update the
|
|
1368
|
+
Update the workspace group definition.
|
|
279
1369
|
|
|
280
1370
|
Parameters
|
|
281
1371
|
----------
|
|
282
1372
|
name : str, optional
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
1373
|
+
Name of the workspace group
|
|
1374
|
+
firewall_ranges : list[str], optional
|
|
1375
|
+
List of allowed CIDR ranges. An empty list indicates that all
|
|
1376
|
+
inbound requests are allowed.
|
|
1377
|
+
admin_password : str, optional
|
|
1378
|
+
Admin password for the workspace group. If no password is supplied,
|
|
1379
|
+
a password will be generated and retured in the response.
|
|
1380
|
+
expires_at : str, optional
|
|
1381
|
+
The timestamp of when the workspace group will expire.
|
|
1382
|
+
If the expiration time is not specified,
|
|
1383
|
+
the workspace group will have no expiration time.
|
|
1384
|
+
At expiration, the workspace group is terminated and all the data is lost.
|
|
1385
|
+
Expiration time can be specified as a timestamp or duration.
|
|
1386
|
+
Example: "2021-01-02T15:04:05Z07:00", "2021-01-02", "3h30m"
|
|
1387
|
+
allow_all_traffic : bool, optional
|
|
1388
|
+
Allow all traffic to the workspace group
|
|
1389
|
+
update_window : Dict[str, int], optional
|
|
1390
|
+
Specify the day and hour of an update window: dict(day=0-6, hour=0-23)
|
|
288
1391
|
|
|
289
1392
|
"""
|
|
290
1393
|
if self._manager is None:
|
|
@@ -293,8 +1396,12 @@ class WorkspaceGroup(object):
|
|
|
293
1396
|
)
|
|
294
1397
|
data = {
|
|
295
1398
|
k: v for k, v in dict(
|
|
296
|
-
name=name,
|
|
1399
|
+
name=name,
|
|
297
1400
|
firewallRanges=firewall_ranges,
|
|
1401
|
+
adminPassword=admin_password,
|
|
1402
|
+
expiresAt=expires_at,
|
|
1403
|
+
allowAllTraffic=allow_all_traffic,
|
|
1404
|
+
updateWindow=snake_to_camel_dict(update_window),
|
|
298
1405
|
).items() if v is not None
|
|
299
1406
|
}
|
|
300
1407
|
self._manager._patch(f'workspaceGroups/{self.id}', json=data)
|
|
@@ -314,7 +1421,7 @@ class WorkspaceGroup(object):
|
|
|
314
1421
|
force : bool, optional
|
|
315
1422
|
Terminate a workspace group even if it has active workspaces
|
|
316
1423
|
wait_on_terminated : bool, optional
|
|
317
|
-
Wait for the
|
|
1424
|
+
Wait for the workspace group to go into 'Terminated' mode before returning
|
|
318
1425
|
wait_interval : int, optional
|
|
319
1426
|
Number of seconds between each server check
|
|
320
1427
|
wait_timeout : int, optional
|
|
@@ -332,15 +1439,26 @@ class WorkspaceGroup(object):
|
|
|
332
1439
|
)
|
|
333
1440
|
self._manager._delete(f'workspaceGroups/{self.id}', params=dict(force=force))
|
|
334
1441
|
if wait_on_terminated:
|
|
335
|
-
|
|
336
|
-
self.
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
1442
|
+
while True:
|
|
1443
|
+
self.refresh()
|
|
1444
|
+
if self.terminated_at is not None:
|
|
1445
|
+
break
|
|
1446
|
+
if wait_timeout <= 0:
|
|
1447
|
+
raise ManagementError(
|
|
1448
|
+
msg='Exceeded waiting time for WorkspaceGroup to terminate',
|
|
1449
|
+
)
|
|
1450
|
+
time.sleep(wait_interval)
|
|
1451
|
+
wait_timeout -= wait_interval
|
|
340
1452
|
|
|
341
1453
|
def create_workspace(
|
|
342
|
-
self,
|
|
343
|
-
|
|
1454
|
+
self,
|
|
1455
|
+
name: str,
|
|
1456
|
+
size: Optional[str] = None,
|
|
1457
|
+
auto_suspend: Optional[Dict[str, Any]] = None,
|
|
1458
|
+
cache_config: Optional[int] = None,
|
|
1459
|
+
enable_kai: Optional[bool] = None,
|
|
1460
|
+
wait_on_active: bool = False,
|
|
1461
|
+
wait_interval: int = 10,
|
|
344
1462
|
wait_timeout: int = 600,
|
|
345
1463
|
) -> Workspace:
|
|
346
1464
|
"""
|
|
@@ -352,6 +1470,15 @@ class WorkspaceGroup(object):
|
|
|
352
1470
|
Name of the workspace
|
|
353
1471
|
size : str, optional
|
|
354
1472
|
Workspace size in workspace size notation (S-00, S-1, etc.)
|
|
1473
|
+
auto_suspend : Dict[str, Any], optional
|
|
1474
|
+
Auto suspend settings for the workspace. If this field is not
|
|
1475
|
+
provided, no settings will be enabled.
|
|
1476
|
+
cache_config : int, optional
|
|
1477
|
+
Specifies the multiplier for the persistent cache associated
|
|
1478
|
+
with the workspace. If specified, it enables the cache configuration
|
|
1479
|
+
multiplier. It can have one of the following values: 1, 2, or 4.
|
|
1480
|
+
enable_kai : bool, optional
|
|
1481
|
+
Whether to create a SingleStore Kai-enabled workspace
|
|
355
1482
|
wait_on_active : bool, optional
|
|
356
1483
|
Wait for the workspace to be active before returning
|
|
357
1484
|
wait_timeout : int, optional
|
|
@@ -369,20 +1496,103 @@ class WorkspaceGroup(object):
|
|
|
369
1496
|
raise ManagementError(
|
|
370
1497
|
msg='No workspace manager is associated with this object.',
|
|
371
1498
|
)
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
1499
|
+
|
|
1500
|
+
out = self._manager.create_workspace(
|
|
1501
|
+
name=name,
|
|
1502
|
+
workspace_group=self,
|
|
1503
|
+
size=size,
|
|
1504
|
+
auto_suspend=snake_to_camel_dict(auto_suspend),
|
|
1505
|
+
cache_config=cache_config,
|
|
1506
|
+
enable_kai=enable_kai,
|
|
1507
|
+
wait_on_active=wait_on_active,
|
|
1508
|
+
wait_interval=wait_interval,
|
|
1509
|
+
wait_timeout=wait_timeout,
|
|
375
1510
|
)
|
|
376
1511
|
|
|
1512
|
+
return out
|
|
1513
|
+
|
|
377
1514
|
@property
|
|
378
|
-
def workspaces(self) ->
|
|
1515
|
+
def workspaces(self) -> NamedList[Workspace]:
|
|
379
1516
|
"""Return a list of available workspaces."""
|
|
380
1517
|
if self._manager is None:
|
|
381
1518
|
raise ManagementError(
|
|
382
1519
|
msg='No workspace manager is associated with this object.',
|
|
383
1520
|
)
|
|
384
1521
|
res = self._manager._get('workspaces', params=dict(workspaceGroupID=self.id))
|
|
385
|
-
return
|
|
1522
|
+
return NamedList(
|
|
1523
|
+
[Workspace.from_dict(item, self._manager) for item in res.json()],
|
|
1524
|
+
)
|
|
1525
|
+
|
|
1526
|
+
|
|
1527
|
+
class Billing(object):
|
|
1528
|
+
"""Billing information."""
|
|
1529
|
+
|
|
1530
|
+
COMPUTE_CREDIT = 'compute_credit'
|
|
1531
|
+
STORAGE_AVG_BYTE = 'storage_avg_byte'
|
|
1532
|
+
|
|
1533
|
+
HOUR = 'hour'
|
|
1534
|
+
DAY = 'day'
|
|
1535
|
+
MONTH = 'month'
|
|
1536
|
+
|
|
1537
|
+
def __init__(self, manager: Manager):
|
|
1538
|
+
self._manager = manager
|
|
1539
|
+
|
|
1540
|
+
def usage(
|
|
1541
|
+
self,
|
|
1542
|
+
start_time: datetime.datetime,
|
|
1543
|
+
end_time: datetime.datetime,
|
|
1544
|
+
metric: Optional[str] = None,
|
|
1545
|
+
aggregate_by: Optional[str] = None,
|
|
1546
|
+
) -> List[BillingUsageItem]:
|
|
1547
|
+
"""
|
|
1548
|
+
Get usage information.
|
|
1549
|
+
|
|
1550
|
+
Parameters
|
|
1551
|
+
----------
|
|
1552
|
+
start_time : datetime.datetime
|
|
1553
|
+
Start time for usage interval
|
|
1554
|
+
end_time : datetime.datetime
|
|
1555
|
+
End time for usage interval
|
|
1556
|
+
metric : str, optional
|
|
1557
|
+
Possible metrics are ``mgr.billing.COMPUTE_CREDIT`` and
|
|
1558
|
+
``mgr.billing.STORAGE_AVG_BYTE`` (default is all)
|
|
1559
|
+
aggregate_by : str, optional
|
|
1560
|
+
Aggregate type used to group usage: ``mgr.billing.HOUR``,
|
|
1561
|
+
``mgr.billing.DAY``, or ``mgr.billing.MONTH``
|
|
1562
|
+
|
|
1563
|
+
Returns
|
|
1564
|
+
-------
|
|
1565
|
+
List[BillingUsage]
|
|
1566
|
+
|
|
1567
|
+
"""
|
|
1568
|
+
res = self._manager._get(
|
|
1569
|
+
'billing/usage',
|
|
1570
|
+
params={
|
|
1571
|
+
k: v for k, v in dict(
|
|
1572
|
+
metric=snake_to_camel(metric),
|
|
1573
|
+
startTime=from_datetime(start_time),
|
|
1574
|
+
endTime=from_datetime(end_time),
|
|
1575
|
+
aggregate_by=aggregate_by.lower() if aggregate_by else None,
|
|
1576
|
+
).items() if v is not None
|
|
1577
|
+
},
|
|
1578
|
+
)
|
|
1579
|
+
return [
|
|
1580
|
+
BillingUsageItem.from_dict(x, self._manager)
|
|
1581
|
+
for x in res.json()['billingUsage']
|
|
1582
|
+
]
|
|
1583
|
+
|
|
1584
|
+
|
|
1585
|
+
class Organizations(object):
|
|
1586
|
+
"""Organizations."""
|
|
1587
|
+
|
|
1588
|
+
def __init__(self, manager: Manager):
|
|
1589
|
+
self._manager = manager
|
|
1590
|
+
|
|
1591
|
+
@property
|
|
1592
|
+
def current(self) -> Organization:
|
|
1593
|
+
"""Get current organization."""
|
|
1594
|
+
res = self._manager._get('organizations/current').json()
|
|
1595
|
+
return Organization.from_dict(res, self._manager)
|
|
386
1596
|
|
|
387
1597
|
|
|
388
1598
|
class WorkspaceManager(Manager):
|
|
@@ -416,20 +1626,44 @@ class WorkspaceManager(Manager):
|
|
|
416
1626
|
obj_type = 'workspace'
|
|
417
1627
|
|
|
418
1628
|
@property
|
|
419
|
-
def workspace_groups(self) ->
|
|
1629
|
+
def workspace_groups(self) -> NamedList[WorkspaceGroup]:
|
|
420
1630
|
"""Return a list of available workspace groups."""
|
|
421
1631
|
res = self._get('workspaceGroups')
|
|
422
|
-
return [WorkspaceGroup.from_dict(item, self) for item in res.json()]
|
|
1632
|
+
return NamedList([WorkspaceGroup.from_dict(item, self) for item in res.json()])
|
|
1633
|
+
|
|
1634
|
+
@property
|
|
1635
|
+
def organizations(self) -> Organizations:
|
|
1636
|
+
"""Return the organizations."""
|
|
1637
|
+
return Organizations(self)
|
|
1638
|
+
|
|
1639
|
+
@property
|
|
1640
|
+
def organization(self) -> Organization:
|
|
1641
|
+
""" Return the current organization."""
|
|
1642
|
+
return self.organizations.current
|
|
423
1643
|
|
|
424
1644
|
@property
|
|
425
|
-
def
|
|
1645
|
+
def billing(self) -> Billing:
|
|
1646
|
+
"""Return the current billing information."""
|
|
1647
|
+
return Billing(self)
|
|
1648
|
+
|
|
1649
|
+
@ttl_property(datetime.timedelta(hours=1))
|
|
1650
|
+
def regions(self) -> NamedList[Region]:
|
|
426
1651
|
"""Return a list of available regions."""
|
|
427
1652
|
res = self._get('regions')
|
|
428
|
-
return [Region.from_dict(item, self) for item in res.json()]
|
|
1653
|
+
return NamedList([Region.from_dict(item, self) for item in res.json()])
|
|
429
1654
|
|
|
430
1655
|
def create_workspace_group(
|
|
431
|
-
self,
|
|
432
|
-
|
|
1656
|
+
self,
|
|
1657
|
+
name: str,
|
|
1658
|
+
region: Union[str, Region],
|
|
1659
|
+
firewall_ranges: List[str],
|
|
1660
|
+
admin_password: Optional[str] = None,
|
|
1661
|
+
backup_bucket_kms_key_id: Optional[str] = None,
|
|
1662
|
+
data_bucket_kms_key_id: Optional[str] = None,
|
|
1663
|
+
expires_at: Optional[str] = None,
|
|
1664
|
+
smart_dr: Optional[bool] = None,
|
|
1665
|
+
allow_all_traffic: Optional[bool] = None,
|
|
1666
|
+
update_window: Optional[Dict[str, int]] = None,
|
|
433
1667
|
) -> WorkspaceGroup:
|
|
434
1668
|
"""
|
|
435
1669
|
Create a new workspace group.
|
|
@@ -446,6 +1680,32 @@ class WorkspaceManager(Manager):
|
|
|
446
1680
|
admin_password : str, optional
|
|
447
1681
|
Admin password for the workspace group. If no password is supplied,
|
|
448
1682
|
a password will be generated and retured in the response.
|
|
1683
|
+
backup_bucket_kms_key_id : str, optional
|
|
1684
|
+
Specifies the KMS key ID associated with the backup bucket.
|
|
1685
|
+
If specified, enables Customer-Managed Encryption Keys (CMEK)
|
|
1686
|
+
encryption for the backup bucket of the workspace group.
|
|
1687
|
+
This feature is only supported in workspace groups deployed in AWS.
|
|
1688
|
+
data_bucket_kms_key_id : str, optional
|
|
1689
|
+
Specifies the KMS key ID associated with the data bucket.
|
|
1690
|
+
If specified, enables Customer-Managed Encryption Keys (CMEK)
|
|
1691
|
+
encryption for the data bucket and Amazon Elastic Block Store
|
|
1692
|
+
(EBS) volumes of the workspace group. This feature is only supported
|
|
1693
|
+
in workspace groups deployed in AWS.
|
|
1694
|
+
expires_at : str, optional
|
|
1695
|
+
The timestamp of when the workspace group will expire.
|
|
1696
|
+
If the expiration time is not specified,
|
|
1697
|
+
the workspace group will have no expiration time.
|
|
1698
|
+
At expiration, the workspace group is terminated and all the data is lost.
|
|
1699
|
+
Expiration time can be specified as a timestamp or duration.
|
|
1700
|
+
Example: "2021-01-02T15:04:05Z07:00", "2021-01-02", "3h30m"
|
|
1701
|
+
smart_dr : bool, optional
|
|
1702
|
+
Enables Smart Disaster Recovery (SmartDR) for the workspace group.
|
|
1703
|
+
SmartDR is a disaster recovery solution that ensures seamless and
|
|
1704
|
+
continuous replication of data from the primary region to a secondary region
|
|
1705
|
+
allow_all_traffic : bool, optional
|
|
1706
|
+
Allow all traffic to the workspace group
|
|
1707
|
+
update_window : Dict[str, int], optional
|
|
1708
|
+
Specify the day and hour of an update window: dict(day=0-6, hour=0-23)
|
|
449
1709
|
|
|
450
1710
|
Returns
|
|
451
1711
|
-------
|
|
@@ -458,15 +1718,28 @@ class WorkspaceManager(Manager):
|
|
|
458
1718
|
'workspaceGroups', json=dict(
|
|
459
1719
|
name=name, regionID=region,
|
|
460
1720
|
adminPassword=admin_password,
|
|
461
|
-
|
|
1721
|
+
backupBucketKMSKeyID=backup_bucket_kms_key_id,
|
|
1722
|
+
dataBucketKMSKeyID=data_bucket_kms_key_id,
|
|
1723
|
+
firewallRanges=firewall_ranges or [],
|
|
1724
|
+
expiresAt=expires_at,
|
|
1725
|
+
smartDR=smart_dr,
|
|
1726
|
+
allowAllTraffic=allow_all_traffic,
|
|
1727
|
+
updateWindow=snake_to_camel_dict(update_window),
|
|
462
1728
|
),
|
|
463
1729
|
)
|
|
464
1730
|
return self.get_workspace_group(res.json()['workspaceGroupID'])
|
|
465
1731
|
|
|
466
1732
|
def create_workspace(
|
|
467
|
-
self,
|
|
468
|
-
|
|
469
|
-
|
|
1733
|
+
self,
|
|
1734
|
+
name: str,
|
|
1735
|
+
workspace_group: Union[str, WorkspaceGroup],
|
|
1736
|
+
size: Optional[str] = None,
|
|
1737
|
+
auto_suspend: Optional[Dict[str, Any]] = None,
|
|
1738
|
+
cache_config: Optional[int] = None,
|
|
1739
|
+
enable_kai: Optional[bool] = None,
|
|
1740
|
+
wait_on_active: bool = False,
|
|
1741
|
+
wait_interval: int = 10,
|
|
1742
|
+
wait_timeout: int = 600,
|
|
470
1743
|
) -> Workspace:
|
|
471
1744
|
"""
|
|
472
1745
|
Create a new workspace.
|
|
@@ -479,6 +1752,15 @@ class WorkspaceManager(Manager):
|
|
|
479
1752
|
The workspace ID of the workspace
|
|
480
1753
|
size : str, optional
|
|
481
1754
|
Workspace size in workspace size notation (S-00, S-1, etc.)
|
|
1755
|
+
auto_suspend : Dict[str, Any], optional
|
|
1756
|
+
Auto suspend settings for the workspace. If this field is not
|
|
1757
|
+
provided, no settings will be enabled.
|
|
1758
|
+
cache_config : int, optional
|
|
1759
|
+
Specifies the multiplier for the persistent cache associated
|
|
1760
|
+
with the workspace. If specified, it enables the cache configuration
|
|
1761
|
+
multiplier. It can have one of the following values: 1, 2, or 4.
|
|
1762
|
+
enable_kai : bool, optional
|
|
1763
|
+
Whether to create a SingleStore Kai-enabled workspace
|
|
482
1764
|
wait_on_active : bool, optional
|
|
483
1765
|
Wait for the workspace to be active before returning
|
|
484
1766
|
wait_timeout : int, optional
|
|
@@ -496,14 +1778,20 @@ class WorkspaceManager(Manager):
|
|
|
496
1778
|
workspace_group = workspace_group.id
|
|
497
1779
|
res = self._post(
|
|
498
1780
|
'workspaces', json=dict(
|
|
499
|
-
name=name,
|
|
1781
|
+
name=name,
|
|
1782
|
+
workspaceGroupID=workspace_group,
|
|
500
1783
|
size=size,
|
|
1784
|
+
autoSuspend=snake_to_camel_dict(auto_suspend),
|
|
1785
|
+
cacheConfig=cache_config,
|
|
1786
|
+
enableKai=enable_kai,
|
|
501
1787
|
),
|
|
502
1788
|
)
|
|
503
1789
|
out = self.get_workspace(res.json()['workspaceID'])
|
|
504
1790
|
if wait_on_active:
|
|
505
1791
|
out = self._wait_on_state(
|
|
506
|
-
out,
|
|
1792
|
+
out,
|
|
1793
|
+
'Active',
|
|
1794
|
+
interval=wait_interval,
|
|
507
1795
|
timeout=wait_timeout,
|
|
508
1796
|
)
|
|
509
1797
|
return out
|
|
@@ -547,6 +1835,8 @@ def manage_workspaces(
|
|
|
547
1835
|
access_token: Optional[str] = None,
|
|
548
1836
|
version: str = WorkspaceManager.default_version,
|
|
549
1837
|
base_url: str = WorkspaceManager.default_base_url,
|
|
1838
|
+
*,
|
|
1839
|
+
organization_id: Optional[str] = None,
|
|
550
1840
|
) -> WorkspaceManager:
|
|
551
1841
|
"""
|
|
552
1842
|
Retrieve a SingleStoreDB workspace manager.
|
|
@@ -559,10 +1849,15 @@ def manage_workspaces(
|
|
|
559
1849
|
Version of the API to use
|
|
560
1850
|
base_url : str, optional
|
|
561
1851
|
Base URL of the workspace management API
|
|
1852
|
+
organization_id : str, optional
|
|
1853
|
+
ID of organization, if using a JWT for authentication
|
|
562
1854
|
|
|
563
1855
|
Returns
|
|
564
1856
|
-------
|
|
565
1857
|
:class:`WorkspaceManager`
|
|
566
1858
|
|
|
567
1859
|
"""
|
|
568
|
-
return WorkspaceManager(
|
|
1860
|
+
return WorkspaceManager(
|
|
1861
|
+
access_token=access_token, base_url=base_url,
|
|
1862
|
+
version=version, organization_id=organization_id,
|
|
1863
|
+
)
|