singlestoredb 0.8.9__cp36-abi3-win_amd64.whl → 0.9.0__cp36-abi3-win_amd64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of singlestoredb might be problematic. Click here for more details.
- _singlestoredb_accel.pyd +0 -0
- singlestoredb/__init__.py +1 -1
- singlestoredb/exceptions.py +24 -0
- singlestoredb/functions/__init__.py +1 -0
- singlestoredb/functions/decorator.py +165 -0
- singlestoredb/functions/dtypes.py +1396 -0
- singlestoredb/functions/ext/__init__.py +2 -0
- singlestoredb/functions/ext/asgi.py +357 -0
- singlestoredb/functions/ext/json.py +49 -0
- singlestoredb/functions/ext/rowdat_1.py +111 -0
- singlestoredb/functions/signature.py +607 -0
- singlestoredb/management/billing_usage.py +148 -0
- singlestoredb/management/manager.py +42 -1
- singlestoredb/management/organization.py +85 -0
- singlestoredb/management/utils.py +118 -1
- singlestoredb/management/workspace.py +881 -5
- singlestoredb/mysql/__init__.py +12 -10
- singlestoredb/mysql/charset.py +12 -11
- singlestoredb/mysql/constants/CLIENT.py +0 -1
- singlestoredb/mysql/constants/COMMAND.py +0 -1
- singlestoredb/mysql/constants/CR.py +0 -2
- singlestoredb/mysql/constants/ER.py +0 -1
- singlestoredb/mysql/constants/FIELD_TYPE.py +0 -1
- singlestoredb/mysql/constants/FLAG.py +0 -1
- singlestoredb/mysql/constants/SERVER_STATUS.py +0 -1
- singlestoredb/mysql/converters.py +49 -28
- singlestoredb/mysql/err.py +3 -3
- singlestoredb/mysql/optionfile.py +4 -4
- singlestoredb/mysql/times.py +3 -4
- singlestoredb/tests/test2.sql +1 -0
- singlestoredb/tests/test_management.py +393 -3
- singlestoredb/tests/test_udf.py +698 -0
- {singlestoredb-0.8.9.dist-info → singlestoredb-0.9.0.dist-info}/METADATA +1 -1
- {singlestoredb-0.8.9.dist-info → singlestoredb-0.9.0.dist-info}/RECORD +37 -25
- {singlestoredb-0.8.9.dist-info → singlestoredb-0.9.0.dist-info}/LICENSE +0 -0
- {singlestoredb-0.8.9.dist-info → singlestoredb-0.9.0.dist-info}/WHEEL +0 -0
- {singlestoredb-0.8.9.dist-info → singlestoredb-0.9.0.dist-info}/top_level.txt +0 -0
|
@@ -1,20 +1,794 @@
|
|
|
1
1
|
#!/usr/bin/env python
|
|
2
2
|
"""SingleStoreDB Workspace Management."""
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
3
5
|
import datetime
|
|
6
|
+
import glob
|
|
7
|
+
import io
|
|
8
|
+
import os
|
|
9
|
+
import pathlib
|
|
10
|
+
import re
|
|
4
11
|
from typing import Any
|
|
12
|
+
from typing import BinaryIO
|
|
5
13
|
from typing import Dict
|
|
6
14
|
from typing import List
|
|
7
15
|
from typing import Optional
|
|
16
|
+
from typing import TextIO
|
|
8
17
|
from typing import Union
|
|
9
18
|
|
|
10
19
|
from .. import connection
|
|
11
20
|
from ..exceptions import ManagementError
|
|
21
|
+
from .billing_usage import BillingUsageItem
|
|
12
22
|
from .manager import Manager
|
|
23
|
+
from .organization import Organization
|
|
13
24
|
from .region import Region
|
|
25
|
+
from .utils import from_datetime
|
|
26
|
+
from .utils import PathLike
|
|
27
|
+
from .utils import snake_to_camel
|
|
14
28
|
from .utils import to_datetime
|
|
15
29
|
from .utils import vars_to_str
|
|
16
30
|
|
|
17
31
|
|
|
32
|
+
class StagesObject(object):
|
|
33
|
+
"""
|
|
34
|
+
Stages file / folder object.
|
|
35
|
+
|
|
36
|
+
This object is not instantiated directly. It is used in the results
|
|
37
|
+
of various operations in ``WorkspaceGroup.stages`` methods.
|
|
38
|
+
|
|
39
|
+
"""
|
|
40
|
+
|
|
41
|
+
def __init__(
|
|
42
|
+
self,
|
|
43
|
+
name: str,
|
|
44
|
+
path: PathLike,
|
|
45
|
+
size: int,
|
|
46
|
+
type: str,
|
|
47
|
+
format: str,
|
|
48
|
+
mimetype: str,
|
|
49
|
+
created: Optional[datetime.datetime],
|
|
50
|
+
last_modified: Optional[datetime.datetime],
|
|
51
|
+
writable: bool,
|
|
52
|
+
content: Optional[List[str]] = None,
|
|
53
|
+
):
|
|
54
|
+
#: Name of file / folder
|
|
55
|
+
self.name = name
|
|
56
|
+
|
|
57
|
+
#: Path of file / folder
|
|
58
|
+
self.path = pathlib.PurePath(path)
|
|
59
|
+
|
|
60
|
+
#: Size of the object (in bytes)
|
|
61
|
+
self.size = size
|
|
62
|
+
|
|
63
|
+
#: Data type: file or directory
|
|
64
|
+
self.type = type
|
|
65
|
+
|
|
66
|
+
#: Data format
|
|
67
|
+
self.format = format
|
|
68
|
+
|
|
69
|
+
#: Mime type
|
|
70
|
+
self.mimetype = mimetype
|
|
71
|
+
|
|
72
|
+
#: Datetime the object was created
|
|
73
|
+
self.created_at = created
|
|
74
|
+
|
|
75
|
+
#: Datetime the object was modified last
|
|
76
|
+
self.last_modified_at = last_modified
|
|
77
|
+
|
|
78
|
+
#: Is the object writable?
|
|
79
|
+
self.writable = writable
|
|
80
|
+
|
|
81
|
+
#: Contents of a directory
|
|
82
|
+
self.content: List[str] = content or []
|
|
83
|
+
|
|
84
|
+
self._stages: Optional[Stages] = None
|
|
85
|
+
|
|
86
|
+
@classmethod
|
|
87
|
+
def from_dict(
|
|
88
|
+
cls,
|
|
89
|
+
obj: Dict[str, Any],
|
|
90
|
+
stages: Stages,
|
|
91
|
+
) -> StagesObject:
|
|
92
|
+
"""
|
|
93
|
+
Construct a StagesObject from a dictionary of values.
|
|
94
|
+
|
|
95
|
+
Parameters
|
|
96
|
+
----------
|
|
97
|
+
obj : dict
|
|
98
|
+
Dictionary of values
|
|
99
|
+
stages : Stages
|
|
100
|
+
Stages object to use as the parent
|
|
101
|
+
|
|
102
|
+
Returns
|
|
103
|
+
-------
|
|
104
|
+
:class:`StagesObject`
|
|
105
|
+
|
|
106
|
+
"""
|
|
107
|
+
out = cls(
|
|
108
|
+
name=obj['name'],
|
|
109
|
+
path=obj['path'],
|
|
110
|
+
size=obj['size'],
|
|
111
|
+
type=obj['type'],
|
|
112
|
+
format=obj['format'],
|
|
113
|
+
mimetype=obj['mimetype'],
|
|
114
|
+
created=to_datetime(obj['created']),
|
|
115
|
+
last_modified=to_datetime(obj['last_modified']),
|
|
116
|
+
writable=bool(obj['writable']),
|
|
117
|
+
)
|
|
118
|
+
out._stages = stages
|
|
119
|
+
return out
|
|
120
|
+
|
|
121
|
+
def __str__(self) -> str:
|
|
122
|
+
"""Return string representation."""
|
|
123
|
+
return vars_to_str(self)
|
|
124
|
+
|
|
125
|
+
def __repr__(self) -> str:
|
|
126
|
+
"""Return string representation."""
|
|
127
|
+
return str(self)
|
|
128
|
+
|
|
129
|
+
def download(
|
|
130
|
+
self,
|
|
131
|
+
local_path: Optional[PathLike] = None,
|
|
132
|
+
*,
|
|
133
|
+
overwrite: bool = False,
|
|
134
|
+
encoding: Optional[str] = None,
|
|
135
|
+
) -> Optional[Union[bytes, str]]:
|
|
136
|
+
"""
|
|
137
|
+
Download the content of a stage path.
|
|
138
|
+
|
|
139
|
+
Parameters
|
|
140
|
+
----------
|
|
141
|
+
local_path : Path or str
|
|
142
|
+
Path to local file target location
|
|
143
|
+
overwrite : bool, optional
|
|
144
|
+
Should an existing file be overwritten if it exists?
|
|
145
|
+
encoding : str, optional
|
|
146
|
+
Encoding used to convert the resulting data
|
|
147
|
+
|
|
148
|
+
Returns
|
|
149
|
+
-------
|
|
150
|
+
bytes or str or None
|
|
151
|
+
|
|
152
|
+
"""
|
|
153
|
+
if self._stages is None:
|
|
154
|
+
raise ManagementError(
|
|
155
|
+
msg='No Stages object is associated with this object.',
|
|
156
|
+
)
|
|
157
|
+
|
|
158
|
+
return self._stages.download(
|
|
159
|
+
self.path, local_path=local_path,
|
|
160
|
+
overwrite=overwrite, encoding=encoding,
|
|
161
|
+
)
|
|
162
|
+
|
|
163
|
+
def remove(self) -> None:
|
|
164
|
+
"""Delete the stage file."""
|
|
165
|
+
if self._stages is None:
|
|
166
|
+
raise ManagementError(
|
|
167
|
+
msg='No Stages object is associated with this object.',
|
|
168
|
+
)
|
|
169
|
+
|
|
170
|
+
self._stages.remove(self.path)
|
|
171
|
+
|
|
172
|
+
def rmdir(self) -> None:
|
|
173
|
+
"""Delete the empty stage directory."""
|
|
174
|
+
if self._stages is None:
|
|
175
|
+
raise ManagementError(
|
|
176
|
+
msg='No Stages object is associated with this object.',
|
|
177
|
+
)
|
|
178
|
+
|
|
179
|
+
self._stages.rmdir(self.path)
|
|
180
|
+
|
|
181
|
+
def removedirs(self) -> None:
|
|
182
|
+
"""Delete the stage directory recursively."""
|
|
183
|
+
if self._stages is None:
|
|
184
|
+
raise ManagementError(
|
|
185
|
+
msg='No Stages object is associated with this object.',
|
|
186
|
+
)
|
|
187
|
+
|
|
188
|
+
self._stages.removedirs(self.path)
|
|
189
|
+
|
|
190
|
+
def rename(self, new_path: PathLike, *, overwrite: bool = False) -> StagesObject:
|
|
191
|
+
"""
|
|
192
|
+
Move the stage file to a new location.
|
|
193
|
+
|
|
194
|
+
Parameters
|
|
195
|
+
----------
|
|
196
|
+
new_path : Path or str
|
|
197
|
+
The new location of the file
|
|
198
|
+
overwrite : bool, optional
|
|
199
|
+
Should path be overwritten if it already exists?
|
|
200
|
+
|
|
201
|
+
"""
|
|
202
|
+
if self._stages is None:
|
|
203
|
+
raise ManagementError(
|
|
204
|
+
msg='No Stages object is associated with this object.',
|
|
205
|
+
)
|
|
206
|
+
return self._stages.rename(self.path, new_path, overwrite=overwrite)
|
|
207
|
+
|
|
208
|
+
def exists(self) -> bool:
|
|
209
|
+
"""Does the file / folder exist?"""
|
|
210
|
+
if self._stages is None:
|
|
211
|
+
raise ManagementError(
|
|
212
|
+
msg='No Stages object is associated with this object.',
|
|
213
|
+
)
|
|
214
|
+
return self._stages.exists(self.path)
|
|
215
|
+
|
|
216
|
+
def is_dir(self) -> bool:
|
|
217
|
+
"""Is the stage object a directory?"""
|
|
218
|
+
return self.type == 'directory'
|
|
219
|
+
|
|
220
|
+
def is_file(self) -> bool:
|
|
221
|
+
"""Is the stage object a file?"""
|
|
222
|
+
return self.type != 'directory'
|
|
223
|
+
|
|
224
|
+
def abspath(self) -> str:
|
|
225
|
+
"""Return the full path of the object."""
|
|
226
|
+
return str(self.path)
|
|
227
|
+
|
|
228
|
+
def basename(self) -> str:
|
|
229
|
+
"""Return the basename of the object."""
|
|
230
|
+
return self.name
|
|
231
|
+
|
|
232
|
+
def dirname(self) -> str:
|
|
233
|
+
"""Return the directory name of the object."""
|
|
234
|
+
return os.path.dirname(self.path)
|
|
235
|
+
|
|
236
|
+
def getmtime(self) -> float:
|
|
237
|
+
"""Return the last modified datetime as a UNIX timestamp."""
|
|
238
|
+
if self.last_modified_at is None:
|
|
239
|
+
return 0.0
|
|
240
|
+
return self.last_modified_at.timestamp()
|
|
241
|
+
|
|
242
|
+
def getctime(self) -> float:
|
|
243
|
+
"""Return the creation datetime as a UNIX timestamp."""
|
|
244
|
+
if self.created_at is None:
|
|
245
|
+
return 0.0
|
|
246
|
+
return self.created_at.timestamp()
|
|
247
|
+
|
|
248
|
+
|
|
249
|
+
class StagesObjectTextWriter(io.StringIO):
|
|
250
|
+
"""StringIO wrapper for writing to Stages."""
|
|
251
|
+
|
|
252
|
+
def __init__(self, buffer: Optional[str], stages: Stages, stage_path: PathLike):
|
|
253
|
+
self._stages = stages
|
|
254
|
+
self._stage_path = stage_path
|
|
255
|
+
super().__init__(buffer)
|
|
256
|
+
|
|
257
|
+
def close(self) -> None:
|
|
258
|
+
"""Write the content to the stage path."""
|
|
259
|
+
print('CLOSING')
|
|
260
|
+
self._stages._upload(self.getvalue(), self._stage_path)
|
|
261
|
+
super().close()
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
class StagesObjectTextReader(io.StringIO):
|
|
265
|
+
"""StringIO wrapper for reading from Stages."""
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
class StagesObjectBytesWriter(io.BytesIO):
|
|
269
|
+
"""BytesIO wrapper for writing to Stages."""
|
|
270
|
+
|
|
271
|
+
def __init__(self, buffer: bytes, stages: Stages, stage_path: PathLike):
|
|
272
|
+
self._stages = stages
|
|
273
|
+
self._stage_path = stage_path
|
|
274
|
+
super().__init__(buffer)
|
|
275
|
+
|
|
276
|
+
def close(self) -> None:
|
|
277
|
+
"""Write the content to the stage path."""
|
|
278
|
+
self._stages._upload(self.getvalue(), self._stage_path)
|
|
279
|
+
super().close()
|
|
280
|
+
|
|
281
|
+
|
|
282
|
+
class StagesObjectBytesReader(io.BytesIO):
|
|
283
|
+
"""BytesIO wrapper for reading from Stages."""
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
class Stages(object):
|
|
287
|
+
"""
|
|
288
|
+
Stages manager.
|
|
289
|
+
|
|
290
|
+
This object is not instantiated directly.
|
|
291
|
+
It is returned by ``WorkspaceGroup.stages``.
|
|
292
|
+
|
|
293
|
+
"""
|
|
294
|
+
|
|
295
|
+
def __init__(self, workspace_group: WorkspaceGroup, manager: WorkspaceManager):
|
|
296
|
+
self._workspace_group = workspace_group
|
|
297
|
+
self._manager = manager
|
|
298
|
+
|
|
299
|
+
def open(
|
|
300
|
+
self,
|
|
301
|
+
stage_path: PathLike,
|
|
302
|
+
mode: str = 'r',
|
|
303
|
+
encoding: Optional[str] = None,
|
|
304
|
+
) -> Union[io.StringIO, io.BytesIO]:
|
|
305
|
+
"""
|
|
306
|
+
Open a Stage path for reading or writing.
|
|
307
|
+
|
|
308
|
+
Parameters
|
|
309
|
+
----------
|
|
310
|
+
stage_path : Path or str
|
|
311
|
+
The stage path to read / write
|
|
312
|
+
mode : str, optional
|
|
313
|
+
The read / write mode. The following modes are supported:
|
|
314
|
+
* 'r' open for reading (default)
|
|
315
|
+
* 'w' open for writing, truncating the file first
|
|
316
|
+
* 'x' create a new file and open it for writing
|
|
317
|
+
The data type can be specified by adding one of the following:
|
|
318
|
+
* 'b' binary mode
|
|
319
|
+
* 't' text mode (default)
|
|
320
|
+
encoding : str, optional
|
|
321
|
+
The string encoding to use for text
|
|
322
|
+
|
|
323
|
+
Returns
|
|
324
|
+
-------
|
|
325
|
+
StagesObjectBytesReader - 'rb' or 'b' mode
|
|
326
|
+
StagesObjectBytesWriter - 'wb' or 'xb' mode
|
|
327
|
+
StagesObjectTextReader - 'r' or 'rt' mode
|
|
328
|
+
StagesObjectTextWriter - 'w', 'x', 'wt' or 'xt' mode
|
|
329
|
+
|
|
330
|
+
"""
|
|
331
|
+
if '+' in mode or 'a' in mode:
|
|
332
|
+
raise ValueError('modifying an existing stage file is not supported')
|
|
333
|
+
|
|
334
|
+
if 'w' in mode or 'x' in mode:
|
|
335
|
+
exists = self.exists(stage_path)
|
|
336
|
+
if exists:
|
|
337
|
+
if 'x' in mode:
|
|
338
|
+
raise FileExistsError(f'stage path already exists: {stage_path}')
|
|
339
|
+
self.remove(stage_path)
|
|
340
|
+
if 'b' in mode:
|
|
341
|
+
return StagesObjectBytesWriter(b'', self, stage_path)
|
|
342
|
+
return StagesObjectTextWriter('', self, stage_path)
|
|
343
|
+
|
|
344
|
+
if 'r' in mode:
|
|
345
|
+
content = self.download(stage_path)
|
|
346
|
+
if isinstance(content, bytes):
|
|
347
|
+
if 'b' in mode:
|
|
348
|
+
return StagesObjectBytesReader(content)
|
|
349
|
+
encoding = 'utf-8' if encoding is None else encoding
|
|
350
|
+
return StagesObjectTextReader(content.decode(encoding))
|
|
351
|
+
|
|
352
|
+
if isinstance(content, str):
|
|
353
|
+
return StagesObjectTextReader(content)
|
|
354
|
+
|
|
355
|
+
raise ValueError(f'unrecognized file content type: {type(content)}')
|
|
356
|
+
|
|
357
|
+
raise ValueError(f'must have one of create/read/write mode specified: {mode}')
|
|
358
|
+
|
|
359
|
+
def upload_file(
|
|
360
|
+
self,
|
|
361
|
+
local_path: PathLike,
|
|
362
|
+
stage_path: PathLike,
|
|
363
|
+
*,
|
|
364
|
+
overwrite: bool = False,
|
|
365
|
+
) -> StagesObject:
|
|
366
|
+
"""
|
|
367
|
+
Upload a local file.
|
|
368
|
+
|
|
369
|
+
Parameters
|
|
370
|
+
----------
|
|
371
|
+
local_path : Path or str
|
|
372
|
+
Path to the local file
|
|
373
|
+
stage_path : Path or str
|
|
374
|
+
Path to the stage file
|
|
375
|
+
overwrite : bool, optional
|
|
376
|
+
Should the ``stage_path`` be overwritten if it exists already?
|
|
377
|
+
|
|
378
|
+
"""
|
|
379
|
+
if not os.path.isfile(local_path):
|
|
380
|
+
raise IsADirectoryError(f'local path is not a file: {local_path}')
|
|
381
|
+
|
|
382
|
+
if self.exists(stage_path):
|
|
383
|
+
if not overwrite:
|
|
384
|
+
raise OSError(f'stage path already exists: {stage_path}')
|
|
385
|
+
|
|
386
|
+
self.remove(stage_path)
|
|
387
|
+
|
|
388
|
+
return self._upload(open(local_path, 'rb'), stage_path, overwrite=overwrite)
|
|
389
|
+
|
|
390
|
+
def upload_folder(
|
|
391
|
+
self,
|
|
392
|
+
local_path: PathLike,
|
|
393
|
+
stage_path: PathLike,
|
|
394
|
+
*,
|
|
395
|
+
overwrite: bool = False,
|
|
396
|
+
recursive: bool = True,
|
|
397
|
+
include_root: bool = False,
|
|
398
|
+
ignore: Optional[Union[PathLike, List[PathLike]]] = None,
|
|
399
|
+
) -> StagesObject:
|
|
400
|
+
"""
|
|
401
|
+
Upload a folder recursively.
|
|
402
|
+
|
|
403
|
+
Only the contents of the folder are uploaded. To include the
|
|
404
|
+
folder name itself in the target path use ``include_root=True``.
|
|
405
|
+
|
|
406
|
+
Parameters
|
|
407
|
+
----------
|
|
408
|
+
local_path : Path or str
|
|
409
|
+
Local directory to upload
|
|
410
|
+
stage_path : Path or str
|
|
411
|
+
Path of stage folder to upload to
|
|
412
|
+
overwrite : bool, optional
|
|
413
|
+
If a file already exists, should it be overwritten?
|
|
414
|
+
recursive : bool, optional
|
|
415
|
+
Should nested folders be uploaded?
|
|
416
|
+
include_root : bool, optional
|
|
417
|
+
Should the local root folder itself be uploaded as the top folder?
|
|
418
|
+
ignore : Path or str or List[Path] or List[str], optional
|
|
419
|
+
Glob patterns of files to ignore, for example, '**/*.pyc` will
|
|
420
|
+
ignore all '*.pyc' files in the directory tree
|
|
421
|
+
|
|
422
|
+
"""
|
|
423
|
+
if not os.path.isdir(local_path):
|
|
424
|
+
raise NotADirectoryError(f'local path is not a directory: {local_path}')
|
|
425
|
+
if self.exists(stage_path) and not self.is_dir(stage_path):
|
|
426
|
+
raise NotADirectoryError(f'stage path is not a directory: {stage_path}')
|
|
427
|
+
|
|
428
|
+
ignore_files = set()
|
|
429
|
+
if ignore:
|
|
430
|
+
if isinstance(ignore, list):
|
|
431
|
+
for item in ignore:
|
|
432
|
+
ignore_files.update(glob.glob(str(item), recursive=recursive))
|
|
433
|
+
else:
|
|
434
|
+
ignore_files.update(glob.glob(str(ignore), recursive=recursive))
|
|
435
|
+
|
|
436
|
+
parent_dir = os.path.basename(os.getcwd())
|
|
437
|
+
|
|
438
|
+
files = glob.glob(os.path.join(local_path, '**'), recursive=recursive)
|
|
439
|
+
|
|
440
|
+
for src in files:
|
|
441
|
+
if ignore_files and src in ignore_files:
|
|
442
|
+
continue
|
|
443
|
+
target = os.path.join(parent_dir, src) if include_root else src
|
|
444
|
+
self.upload_file(src, target, overwrite=overwrite)
|
|
445
|
+
|
|
446
|
+
return self.info(stage_path)
|
|
447
|
+
|
|
448
|
+
def _upload(
|
|
449
|
+
self,
|
|
450
|
+
content: Union[str, bytes, TextIO, BinaryIO],
|
|
451
|
+
stage_path: PathLike,
|
|
452
|
+
*,
|
|
453
|
+
overwrite: bool = False,
|
|
454
|
+
) -> StagesObject:
|
|
455
|
+
"""
|
|
456
|
+
Upload content to a stage file.
|
|
457
|
+
|
|
458
|
+
Parameters
|
|
459
|
+
----------
|
|
460
|
+
content : str or bytes or file-like
|
|
461
|
+
Content to upload to stage
|
|
462
|
+
stage_path : Path or str
|
|
463
|
+
Path to the stage file
|
|
464
|
+
overwrite : bool, optional
|
|
465
|
+
Should the ``stage_path`` be overwritten if it exists already?
|
|
466
|
+
|
|
467
|
+
"""
|
|
468
|
+
if self.exists(stage_path):
|
|
469
|
+
if not overwrite:
|
|
470
|
+
raise OSError(f'stage path already exists: {stage_path}')
|
|
471
|
+
self.remove(stage_path)
|
|
472
|
+
|
|
473
|
+
self._manager._put(
|
|
474
|
+
f'stages/{self._workspace_group.id}/fs/{stage_path}',
|
|
475
|
+
files={'file': content},
|
|
476
|
+
headers={'Content-Type': None},
|
|
477
|
+
)
|
|
478
|
+
|
|
479
|
+
return self.info(stage_path)
|
|
480
|
+
|
|
481
|
+
def mkdir(self, stage_path: PathLike, overwrite: bool = False) -> StagesObject:
|
|
482
|
+
"""
|
|
483
|
+
Make a directory in the stage.
|
|
484
|
+
|
|
485
|
+
Parameters
|
|
486
|
+
----------
|
|
487
|
+
stage_path : Path or str
|
|
488
|
+
Path of the folder to create
|
|
489
|
+
overwrite : bool, optional
|
|
490
|
+
Should the stage path be overwritten if it exists already?
|
|
491
|
+
|
|
492
|
+
Returns
|
|
493
|
+
-------
|
|
494
|
+
StagesObject
|
|
495
|
+
|
|
496
|
+
"""
|
|
497
|
+
if self.exists(stage_path):
|
|
498
|
+
if not overwrite:
|
|
499
|
+
return self.info(stage_path)
|
|
500
|
+
|
|
501
|
+
self.remove(stage_path)
|
|
502
|
+
|
|
503
|
+
self._manager._put(
|
|
504
|
+
f'stages/{self._workspace_group.id}/fs/{stage_path}',
|
|
505
|
+
)
|
|
506
|
+
|
|
507
|
+
return self.info(stage_path)
|
|
508
|
+
|
|
509
|
+
mkdirs = mkdir
|
|
510
|
+
|
|
511
|
+
def rename(
|
|
512
|
+
self,
|
|
513
|
+
old_path: PathLike,
|
|
514
|
+
new_path: PathLike,
|
|
515
|
+
*,
|
|
516
|
+
overwrite: bool = False,
|
|
517
|
+
) -> StagesObject:
|
|
518
|
+
"""
|
|
519
|
+
Move the stage file to a new location.
|
|
520
|
+
|
|
521
|
+
Paraemeters
|
|
522
|
+
-----------
|
|
523
|
+
old_path : Path or str
|
|
524
|
+
Original location of the path
|
|
525
|
+
new_path : Path or str
|
|
526
|
+
New location of the path
|
|
527
|
+
overwrite : bool, optional
|
|
528
|
+
Should the ``new_path`` be overwritten if it exists already?
|
|
529
|
+
|
|
530
|
+
"""
|
|
531
|
+
if not self.exists(old_path):
|
|
532
|
+
raise OSError(f'stage path does not exist: {old_path}')
|
|
533
|
+
|
|
534
|
+
if self.exists(new_path):
|
|
535
|
+
if not overwrite:
|
|
536
|
+
raise OSError(f'stage path already exists: {new_path}')
|
|
537
|
+
|
|
538
|
+
self.remove(new_path)
|
|
539
|
+
|
|
540
|
+
self._manager._patch(
|
|
541
|
+
f'stages/{self._workspace_group.id}/fs/{old_path}',
|
|
542
|
+
json=dict(newPath=new_path),
|
|
543
|
+
)
|
|
544
|
+
|
|
545
|
+
return self.info(new_path)
|
|
546
|
+
|
|
547
|
+
def info(self, stage_path: PathLike) -> StagesObject:
|
|
548
|
+
"""
|
|
549
|
+
Return information about a stage location.
|
|
550
|
+
|
|
551
|
+
Parameters
|
|
552
|
+
----------
|
|
553
|
+
stage_path : Path or str
|
|
554
|
+
Path to the stage location
|
|
555
|
+
|
|
556
|
+
Returns
|
|
557
|
+
-------
|
|
558
|
+
StagesObject
|
|
559
|
+
|
|
560
|
+
"""
|
|
561
|
+
res = self._manager._get(
|
|
562
|
+
f'stages/{self._workspace_group.id}/fs/{stage_path}',
|
|
563
|
+
params=dict(metadata=1),
|
|
564
|
+
).json()
|
|
565
|
+
|
|
566
|
+
return StagesObject.from_dict(res, self)
|
|
567
|
+
|
|
568
|
+
def exists(self, stage_path: PathLike) -> bool:
|
|
569
|
+
"""
|
|
570
|
+
Does the given stage path exist?
|
|
571
|
+
|
|
572
|
+
Parameters
|
|
573
|
+
----------
|
|
574
|
+
stage_path : Path or str
|
|
575
|
+
Path to stage object
|
|
576
|
+
|
|
577
|
+
Returns
|
|
578
|
+
-------
|
|
579
|
+
bool
|
|
580
|
+
|
|
581
|
+
"""
|
|
582
|
+
try:
|
|
583
|
+
self.info(stage_path)
|
|
584
|
+
return True
|
|
585
|
+
except ManagementError as exc:
|
|
586
|
+
if 'NoSuchKey' in str(exc):
|
|
587
|
+
return False
|
|
588
|
+
raise
|
|
589
|
+
|
|
590
|
+
def is_dir(self, stage_path: PathLike) -> bool:
|
|
591
|
+
"""
|
|
592
|
+
Is the given stage path a directory?
|
|
593
|
+
|
|
594
|
+
Parameters
|
|
595
|
+
----------
|
|
596
|
+
stage_path : Path or str
|
|
597
|
+
Path to stage object
|
|
598
|
+
|
|
599
|
+
Returns
|
|
600
|
+
-------
|
|
601
|
+
bool
|
|
602
|
+
|
|
603
|
+
"""
|
|
604
|
+
try:
|
|
605
|
+
return self.info(stage_path).type == 'directory'
|
|
606
|
+
except ManagementError as exc:
|
|
607
|
+
if 'NoSuchKey' in str(exc):
|
|
608
|
+
return False
|
|
609
|
+
raise
|
|
610
|
+
|
|
611
|
+
def is_file(self, stage_path: PathLike) -> bool:
|
|
612
|
+
"""
|
|
613
|
+
Is the given stage path a file?
|
|
614
|
+
|
|
615
|
+
Parameters
|
|
616
|
+
----------
|
|
617
|
+
stage_path : Path or str
|
|
618
|
+
Path to stage object
|
|
619
|
+
|
|
620
|
+
Returns
|
|
621
|
+
-------
|
|
622
|
+
bool
|
|
623
|
+
|
|
624
|
+
"""
|
|
625
|
+
try:
|
|
626
|
+
return self.info(stage_path).type != 'directory'
|
|
627
|
+
except ManagementError as exc:
|
|
628
|
+
if 'NoSuchKey' in str(exc):
|
|
629
|
+
return False
|
|
630
|
+
raise
|
|
631
|
+
|
|
632
|
+
def _listdir(self, stage_path: PathLike, *, recursive: bool = False) -> List[str]:
|
|
633
|
+
"""
|
|
634
|
+
Return the names of files in a directory.
|
|
635
|
+
|
|
636
|
+
Parameters
|
|
637
|
+
----------
|
|
638
|
+
stage_path : Path or str
|
|
639
|
+
Path to the folder in Stages
|
|
640
|
+
recursive : bool, optional
|
|
641
|
+
Should folders be listed recursively?
|
|
642
|
+
|
|
643
|
+
"""
|
|
644
|
+
res = self._manager._get(
|
|
645
|
+
f'stages/{self._workspace_group.id}/fs/{stage_path}',
|
|
646
|
+
).json()
|
|
647
|
+
if recursive:
|
|
648
|
+
out = []
|
|
649
|
+
for item in res['content'] or []:
|
|
650
|
+
out.append(item['path'])
|
|
651
|
+
if item['type'] == 'directory':
|
|
652
|
+
out.extend(self._listdir(item['path'], recursive=recursive))
|
|
653
|
+
return out
|
|
654
|
+
return [x['path'] for x in res['content'] or []]
|
|
655
|
+
|
|
656
|
+
def listdir(self, stage_path: PathLike, *, recursive: bool = False) -> List[str]:
|
|
657
|
+
"""
|
|
658
|
+
List the files / folders at the given path.
|
|
659
|
+
|
|
660
|
+
Parameters
|
|
661
|
+
----------
|
|
662
|
+
stage_path : Path or str
|
|
663
|
+
Path to the stage location
|
|
664
|
+
|
|
665
|
+
Returns
|
|
666
|
+
-------
|
|
667
|
+
List[str]
|
|
668
|
+
|
|
669
|
+
"""
|
|
670
|
+
stage_path = re.sub(r'^(\./|/)+', r'', str(stage_path))
|
|
671
|
+
stage_path = re.sub(r'/+$', r'', stage_path)
|
|
672
|
+
|
|
673
|
+
info = self.info(stage_path)
|
|
674
|
+
if info.type == 'directory':
|
|
675
|
+
out = self._listdir(stage_path, recursive=recursive)
|
|
676
|
+
if stage_path:
|
|
677
|
+
stages_path_n = len(stage_path.split('/'))
|
|
678
|
+
out = ['/'.join(x.split('/')[stages_path_n:]) for x in out]
|
|
679
|
+
return out
|
|
680
|
+
|
|
681
|
+
raise NotADirectoryError(f'stage path is not a directory: {stage_path}')
|
|
682
|
+
|
|
683
|
+
def download(
|
|
684
|
+
self,
|
|
685
|
+
stage_path: PathLike,
|
|
686
|
+
local_path: Optional[PathLike] = None,
|
|
687
|
+
*,
|
|
688
|
+
overwrite: bool = False,
|
|
689
|
+
encoding: Optional[str] = None,
|
|
690
|
+
) -> Optional[Union[bytes, str]]:
|
|
691
|
+
"""
|
|
692
|
+
Download the content of a stage path.
|
|
693
|
+
|
|
694
|
+
Parameters
|
|
695
|
+
----------
|
|
696
|
+
stage_path : Path or str
|
|
697
|
+
Path to the stage file
|
|
698
|
+
local_path : Path or str
|
|
699
|
+
Path to local file target location
|
|
700
|
+
overwrite : bool, optional
|
|
701
|
+
Should an existing file be overwritten if it exists?
|
|
702
|
+
encoding : str, optional
|
|
703
|
+
Encoding used to convert the resulting data
|
|
704
|
+
|
|
705
|
+
Returns
|
|
706
|
+
-------
|
|
707
|
+
bytes or str - ``local_path`` is None
|
|
708
|
+
None - ``local_path`` is a Path or str
|
|
709
|
+
|
|
710
|
+
"""
|
|
711
|
+
if local_path is not None and not overwrite and os.path.exists(local_path):
|
|
712
|
+
raise OSError('target file already exists; use overwrite=True to replace')
|
|
713
|
+
if self.is_dir(stage_path):
|
|
714
|
+
raise IsADirectoryError(f'stage path is a directory: {stage_path}')
|
|
715
|
+
|
|
716
|
+
out = self._manager._get(
|
|
717
|
+
f'stages/{self._workspace_group.id}/fs/{stage_path}',
|
|
718
|
+
).content
|
|
719
|
+
|
|
720
|
+
if local_path is not None:
|
|
721
|
+
with open(local_path, 'wb') as outfile:
|
|
722
|
+
outfile.write(out)
|
|
723
|
+
return None
|
|
724
|
+
|
|
725
|
+
if encoding:
|
|
726
|
+
return out.decode(encoding)
|
|
727
|
+
|
|
728
|
+
return out
|
|
729
|
+
|
|
730
|
+
def remove(self, stage_path: PathLike) -> None:
|
|
731
|
+
"""
|
|
732
|
+
Delete a stage location.
|
|
733
|
+
|
|
734
|
+
Parameters
|
|
735
|
+
----------
|
|
736
|
+
stage_path : Path or str
|
|
737
|
+
Path to the stage location
|
|
738
|
+
|
|
739
|
+
"""
|
|
740
|
+
if self.is_dir(stage_path):
|
|
741
|
+
raise IsADirectoryError(
|
|
742
|
+
'stage path is a directory, '
|
|
743
|
+
f'use rmdir or removedirs: {stage_path}',
|
|
744
|
+
)
|
|
745
|
+
|
|
746
|
+
self._manager._delete(f'stages/{self._workspace_group.id}/fs/{stage_path}')
|
|
747
|
+
|
|
748
|
+
def removedirs(self, stage_path: PathLike) -> None:
|
|
749
|
+
"""
|
|
750
|
+
Delete a stage folder recursively.
|
|
751
|
+
|
|
752
|
+
Parameters
|
|
753
|
+
----------
|
|
754
|
+
stage_path : Path or str
|
|
755
|
+
Path to the stage location
|
|
756
|
+
|
|
757
|
+
"""
|
|
758
|
+
info = self.info(stage_path)
|
|
759
|
+
if info.type != 'directory':
|
|
760
|
+
raise NotADirectoryError(f'stage path is not a directory: {stage_path}')
|
|
761
|
+
|
|
762
|
+
self._manager._delete(f'stages/{self._workspace_group.id}/fs/{stage_path}')
|
|
763
|
+
|
|
764
|
+
def rmdir(self, stage_path: PathLike) -> None:
|
|
765
|
+
"""
|
|
766
|
+
Delete a stage folder.
|
|
767
|
+
|
|
768
|
+
Parameters
|
|
769
|
+
----------
|
|
770
|
+
stage_path : Path or str
|
|
771
|
+
Path to the stage location
|
|
772
|
+
|
|
773
|
+
"""
|
|
774
|
+
info = self.info(stage_path)
|
|
775
|
+
if info.type != 'directory':
|
|
776
|
+
raise NotADirectoryError(f'stage path is not a directory: {stage_path}')
|
|
777
|
+
|
|
778
|
+
if self.listdir(stage_path):
|
|
779
|
+
raise OSError(f'stage folder is not empty, use removedirs: {stage_path}')
|
|
780
|
+
|
|
781
|
+
self._manager._delete(f'stages/{self._workspace_group.id}/fs/{stage_path}')
|
|
782
|
+
|
|
783
|
+
def __str__(self) -> str:
|
|
784
|
+
"""Return string representation."""
|
|
785
|
+
return vars_to_str(self)
|
|
786
|
+
|
|
787
|
+
def __repr__(self) -> str:
|
|
788
|
+
"""Return string representation."""
|
|
789
|
+
return str(self)
|
|
790
|
+
|
|
791
|
+
|
|
18
792
|
class Workspace(object):
|
|
19
793
|
"""
|
|
20
794
|
SingleStoreDB workspace definition.
|
|
@@ -106,7 +880,7 @@ class Workspace(object):
|
|
|
106
880
|
out._manager = manager
|
|
107
881
|
return out
|
|
108
882
|
|
|
109
|
-
def refresh(self) ->
|
|
883
|
+
def refresh(self) -> Workspace:
|
|
110
884
|
"""Update the object to the current state."""
|
|
111
885
|
if self._manager is None:
|
|
112
886
|
raise ManagementError(
|
|
@@ -253,7 +1027,8 @@ class WorkspaceGroup(object):
|
|
|
253
1027
|
except IndexError:
|
|
254
1028
|
region = None
|
|
255
1029
|
out = cls(
|
|
256
|
-
name=obj['name'],
|
|
1030
|
+
name=obj['name'],
|
|
1031
|
+
id=obj['workspaceGroupID'],
|
|
257
1032
|
created_at=obj['createdAt'],
|
|
258
1033
|
region=region,
|
|
259
1034
|
firewall_ranges=obj.get('firewallRanges', []),
|
|
@@ -262,6 +1037,15 @@ class WorkspaceGroup(object):
|
|
|
262
1037
|
out._manager = manager
|
|
263
1038
|
return out
|
|
264
1039
|
|
|
1040
|
+
@property
|
|
1041
|
+
def stages(self) -> Stages:
|
|
1042
|
+
"""Stages manager."""
|
|
1043
|
+
if self._manager is None:
|
|
1044
|
+
raise ManagementError(
|
|
1045
|
+
msg='No workspace manager is associated with this object.',
|
|
1046
|
+
)
|
|
1047
|
+
return Stages(self, self._manager)
|
|
1048
|
+
|
|
265
1049
|
def refresh(self) -> 'WorkspaceGroup':
|
|
266
1050
|
"""Update teh object to the current state."""
|
|
267
1051
|
if self._manager is None:
|
|
@@ -389,6 +1173,77 @@ class WorkspaceGroup(object):
|
|
|
389
1173
|
return [Workspace.from_dict(item, self._manager) for item in res.json()]
|
|
390
1174
|
|
|
391
1175
|
|
|
1176
|
+
class Billing(object):
|
|
1177
|
+
"""Billing information."""
|
|
1178
|
+
|
|
1179
|
+
COMPUTE_CREDIT = 'compute_credit'
|
|
1180
|
+
STORAGE_AVG_BYTE = 'storage_avg_byte'
|
|
1181
|
+
|
|
1182
|
+
HOUR = 'hour'
|
|
1183
|
+
DAY = 'day'
|
|
1184
|
+
MONTH = 'month'
|
|
1185
|
+
|
|
1186
|
+
def __init__(self, manager: Manager):
|
|
1187
|
+
self._manager = manager
|
|
1188
|
+
|
|
1189
|
+
def usage(
|
|
1190
|
+
self,
|
|
1191
|
+
start_time: datetime.datetime,
|
|
1192
|
+
end_time: datetime.datetime,
|
|
1193
|
+
metric: Optional[str] = None,
|
|
1194
|
+
aggregate_by: Optional[str] = None,
|
|
1195
|
+
) -> List[BillingUsageItem]:
|
|
1196
|
+
"""
|
|
1197
|
+
Get usage information.
|
|
1198
|
+
|
|
1199
|
+
Parameters
|
|
1200
|
+
----------
|
|
1201
|
+
start_time : datetime.datetime
|
|
1202
|
+
Start time for usage interval
|
|
1203
|
+
end_time : datetime.datetime
|
|
1204
|
+
End time for usage interval
|
|
1205
|
+
metric : str, optional
|
|
1206
|
+
Possible metrics are ``mgr.billing.COMPUTE_CREDIT`` and
|
|
1207
|
+
``mgr.billing.STORAGE_AVG_BYTE`` (default is all)
|
|
1208
|
+
aggregate_by : str, optional
|
|
1209
|
+
Aggregate type used to group usage: ``mgr.billing.HOUR``,
|
|
1210
|
+
``mgr.billing.DAY``, or ``mgr.billing.MONTH``
|
|
1211
|
+
|
|
1212
|
+
Returns
|
|
1213
|
+
-------
|
|
1214
|
+
List[BillingUsage]
|
|
1215
|
+
|
|
1216
|
+
"""
|
|
1217
|
+
res = self._manager._get(
|
|
1218
|
+
'billing/usage',
|
|
1219
|
+
params={
|
|
1220
|
+
k: v for k, v in dict(
|
|
1221
|
+
metric=snake_to_camel(metric),
|
|
1222
|
+
startTime=from_datetime(start_time),
|
|
1223
|
+
endTime=from_datetime(end_time),
|
|
1224
|
+
aggregate_by=aggregate_by.lower() if aggregate_by else None,
|
|
1225
|
+
).items() if v is not None
|
|
1226
|
+
},
|
|
1227
|
+
)
|
|
1228
|
+
return [
|
|
1229
|
+
BillingUsageItem.from_dict(x, self._manager)
|
|
1230
|
+
for x in res.json()['billingUsage']
|
|
1231
|
+
]
|
|
1232
|
+
|
|
1233
|
+
|
|
1234
|
+
class Organizations(object):
|
|
1235
|
+
"""Organizations."""
|
|
1236
|
+
|
|
1237
|
+
def __init__(self, manager: Manager):
|
|
1238
|
+
self._manager = manager
|
|
1239
|
+
|
|
1240
|
+
@property
|
|
1241
|
+
def current(self) -> Organization:
|
|
1242
|
+
"""Get current organization."""
|
|
1243
|
+
res = self._manager._get('organizations/current').json()
|
|
1244
|
+
return Organization.from_dict(res, self._manager)
|
|
1245
|
+
|
|
1246
|
+
|
|
392
1247
|
class WorkspaceManager(Manager):
|
|
393
1248
|
"""
|
|
394
1249
|
SingleStoreDB workspace manager.
|
|
@@ -419,13 +1274,23 @@ class WorkspaceManager(Manager):
|
|
|
419
1274
|
#: Object type
|
|
420
1275
|
obj_type = 'workspace'
|
|
421
1276
|
|
|
422
|
-
@property
|
|
1277
|
+
@ property
|
|
423
1278
|
def workspace_groups(self) -> List[WorkspaceGroup]:
|
|
424
1279
|
"""Return a list of available workspace groups."""
|
|
425
1280
|
res = self._get('workspaceGroups')
|
|
426
1281
|
return [WorkspaceGroup.from_dict(item, self) for item in res.json()]
|
|
427
1282
|
|
|
428
|
-
@property
|
|
1283
|
+
@ property
|
|
1284
|
+
def organizations(self) -> Organizations:
|
|
1285
|
+
"""Return the organizations."""
|
|
1286
|
+
return Organizations(self)
|
|
1287
|
+
|
|
1288
|
+
@ property
|
|
1289
|
+
def billing(self) -> Billing:
|
|
1290
|
+
"""Return the current billing information."""
|
|
1291
|
+
return Billing(self)
|
|
1292
|
+
|
|
1293
|
+
@ property
|
|
429
1294
|
def regions(self) -> List[Region]:
|
|
430
1295
|
"""Return a list of available regions."""
|
|
431
1296
|
res = self._get('regions')
|
|
@@ -435,6 +1300,8 @@ class WorkspaceManager(Manager):
|
|
|
435
1300
|
self, name: str, region: Union[str, Region],
|
|
436
1301
|
firewall_ranges: List[str], admin_password: Optional[str] = None,
|
|
437
1302
|
expires_at: Optional[str] = None,
|
|
1303
|
+
allow_all_traffic: Optional[bool] = None,
|
|
1304
|
+
update_window: Optional[Dict[str, int]] = None,
|
|
438
1305
|
) -> WorkspaceGroup:
|
|
439
1306
|
"""
|
|
440
1307
|
Create a new workspace group.
|
|
@@ -458,6 +1325,10 @@ class WorkspaceManager(Manager):
|
|
|
458
1325
|
At expiration, the workspace group is terminated and all the data is lost.
|
|
459
1326
|
Expiration time can be specified as a timestamp or duration.
|
|
460
1327
|
Example: "2021-01-02T15:04:05Z07:00", "2021-01-02", "3h30m"
|
|
1328
|
+
allow_all_traffic : bool, optional
|
|
1329
|
+
Allow all traffic to the workspace group
|
|
1330
|
+
update_window : Dict[str, int], optional
|
|
1331
|
+
Specify the day and hour of an update window: dict(day=0-6, hour=0-23)
|
|
461
1332
|
|
|
462
1333
|
Returns
|
|
463
1334
|
-------
|
|
@@ -472,6 +1343,8 @@ class WorkspaceManager(Manager):
|
|
|
472
1343
|
adminPassword=admin_password,
|
|
473
1344
|
firewallRanges=firewall_ranges,
|
|
474
1345
|
expiresAt=expires_at,
|
|
1346
|
+
allowAllTraffic=allow_all_traffic,
|
|
1347
|
+
updateWindow=update_window,
|
|
475
1348
|
),
|
|
476
1349
|
)
|
|
477
1350
|
return self.get_workspace_group(res.json()['workspaceGroupID'])
|
|
@@ -578,4 +1451,7 @@ def manage_workspaces(
|
|
|
578
1451
|
:class:`WorkspaceManager`
|
|
579
1452
|
|
|
580
1453
|
"""
|
|
581
|
-
return WorkspaceManager(
|
|
1454
|
+
return WorkspaceManager(
|
|
1455
|
+
access_token=access_token, base_url=base_url,
|
|
1456
|
+
version=version,
|
|
1457
|
+
)
|