singlestoredb 1.12.0__cp38-abi3-win_amd64.whl → 1.12.2__cp38-abi3-win_amd64.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of singlestoredb might be problematic. Click here for more details.
- _singlestoredb_accel.pyd +0 -0
- singlestoredb/__init__.py +1 -1
- singlestoredb/fusion/handlers/utils.py +41 -27
- singlestoredb/management/files.py +8 -8
- singlestoredb/management/workspace.py +5 -6
- singlestoredb/tests/test_fusion.py +808 -0
- singlestoredb/tests/test_management.py +50 -2
- {singlestoredb-1.12.0.dist-info → singlestoredb-1.12.2.dist-info}/METADATA +1 -1
- {singlestoredb-1.12.0.dist-info → singlestoredb-1.12.2.dist-info}/RECORD +13 -13
- {singlestoredb-1.12.0.dist-info → singlestoredb-1.12.2.dist-info}/LICENSE +0 -0
- {singlestoredb-1.12.0.dist-info → singlestoredb-1.12.2.dist-info}/WHEEL +0 -0
- {singlestoredb-1.12.0.dist-info → singlestoredb-1.12.2.dist-info}/entry_points.txt +0 -0
- {singlestoredb-1.12.0.dist-info → singlestoredb-1.12.2.dist-info}/top_level.txt +0 -0
_singlestoredb_accel.pyd
CHANGED
|
Binary file
|
singlestoredb/__init__.py
CHANGED
|
@@ -7,13 +7,11 @@ from typing import Optional
|
|
|
7
7
|
from typing import Union
|
|
8
8
|
|
|
9
9
|
from ...exceptions import ManagementError
|
|
10
|
+
from ...management import files as mgmt_files
|
|
10
11
|
from ...management import manage_workspaces
|
|
11
12
|
from ...management.files import FilesManager
|
|
12
13
|
from ...management.files import FileSpace
|
|
13
14
|
from ...management.files import manage_files
|
|
14
|
-
from ...management.files import MODELS_SPACE
|
|
15
|
-
from ...management.files import PERSONAL_SPACE
|
|
16
|
-
from ...management.files import SHARED_SPACE
|
|
17
15
|
from ...management.workspace import StarterWorkspace
|
|
18
16
|
from ...management.workspace import Workspace
|
|
19
17
|
from ...management.workspace import WorkspaceGroup
|
|
@@ -190,6 +188,10 @@ def get_deployment(
|
|
|
190
188
|
* params['group']['deployment_id']
|
|
191
189
|
* params['in_deployment']['deployment_name']
|
|
192
190
|
* params['in_deployment']['deployment_id']
|
|
191
|
+
* params['in']['in_group']['deployment_name']
|
|
192
|
+
* params['in']['in_group']['deployment_id']
|
|
193
|
+
* params['in']['in_deployment']['deployment_name']
|
|
194
|
+
* params['in']['in_deployment']['deployment_id']
|
|
193
195
|
|
|
194
196
|
Or, from the SINGLESTOREDB_WORKSPACE_GROUP
|
|
195
197
|
or SINGLESTOREDB_CLUSTER environment variables.
|
|
@@ -197,55 +199,65 @@ def get_deployment(
|
|
|
197
199
|
"""
|
|
198
200
|
manager = get_workspace_manager()
|
|
199
201
|
|
|
202
|
+
#
|
|
203
|
+
# Search for deployment by name
|
|
204
|
+
#
|
|
200
205
|
deployment_name = params.get('deployment_name') or \
|
|
201
206
|
(params.get('in_deployment') or {}).get('deployment_name') or \
|
|
202
|
-
(params.get('group') or {}).get('deployment_name')
|
|
207
|
+
(params.get('group') or {}).get('deployment_name') or \
|
|
208
|
+
((params.get('in') or {}).get('in_group') or {}).get('deployment_name') or \
|
|
209
|
+
((params.get('in') or {}).get('in_deployment') or {}).get('deployment_name')
|
|
210
|
+
|
|
203
211
|
if deployment_name:
|
|
212
|
+
# Standard workspace group
|
|
204
213
|
workspace_groups = [
|
|
205
214
|
x for x in manager.workspace_groups
|
|
206
215
|
if x.name == deployment_name
|
|
207
216
|
]
|
|
208
217
|
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
filtered_starter_workspaces = [
|
|
212
|
-
x for x in manager.starter_workspaces
|
|
213
|
-
if x.name == deployment_name
|
|
214
|
-
]
|
|
215
|
-
|
|
216
|
-
if not filtered_starter_workspaces:
|
|
217
|
-
raise KeyError(
|
|
218
|
-
f'no deployment found with name: {deployment_name}',
|
|
219
|
-
)
|
|
220
|
-
|
|
221
|
-
starter_workspaces = filtered_starter_workspaces
|
|
218
|
+
if len(workspace_groups) == 1:
|
|
219
|
+
return workspace_groups[0]
|
|
222
220
|
|
|
223
|
-
|
|
221
|
+
elif len(workspace_groups) > 1:
|
|
224
222
|
ids = ', '.join(x.id for x in workspace_groups)
|
|
225
223
|
raise ValueError(
|
|
226
224
|
f'more than one workspace group with given name was found: {ids}',
|
|
227
225
|
)
|
|
228
226
|
|
|
229
|
-
|
|
227
|
+
# Starter workspace
|
|
228
|
+
starter_workspaces = [
|
|
229
|
+
x for x in manager.starter_workspaces
|
|
230
|
+
if x.name == deployment_name
|
|
231
|
+
]
|
|
232
|
+
|
|
233
|
+
if len(starter_workspaces) == 1:
|
|
234
|
+
return starter_workspaces[0]
|
|
235
|
+
|
|
236
|
+
elif len(starter_workspaces) > 1:
|
|
230
237
|
ids = ', '.join(x.id for x in starter_workspaces)
|
|
231
238
|
raise ValueError(
|
|
232
239
|
f'more than one starter workspace with given name was found: {ids}',
|
|
233
240
|
)
|
|
234
241
|
|
|
235
|
-
|
|
236
|
-
return workspace_groups[0]
|
|
237
|
-
else:
|
|
238
|
-
return starter_workspaces[0]
|
|
242
|
+
raise KeyError(f'no deployment found with name: {deployment_name}')
|
|
239
243
|
|
|
244
|
+
#
|
|
245
|
+
# Search for deployment by ID
|
|
246
|
+
#
|
|
240
247
|
deployment_id = params.get('deployment_id') or \
|
|
241
248
|
(params.get('in_deployment') or {}).get('deployment_id') or \
|
|
242
|
-
(params.get('group') or {}).get('deployment_id')
|
|
249
|
+
(params.get('group') or {}).get('deployment_id') or \
|
|
250
|
+
((params.get('in') or {}).get('in_group') or {}).get('deployment_id') or \
|
|
251
|
+
((params.get('in') or {}).get('in_deployment') or {}).get('deployment_id')
|
|
252
|
+
|
|
243
253
|
if deployment_id:
|
|
244
254
|
try:
|
|
255
|
+
# Standard workspace group
|
|
245
256
|
return manager.get_workspace_group(deployment_id)
|
|
246
257
|
except ManagementError as exc:
|
|
247
258
|
if exc.errno == 404:
|
|
248
259
|
try:
|
|
260
|
+
# Starter workspace
|
|
249
261
|
return manager.get_starter_workspace(deployment_id)
|
|
250
262
|
except ManagementError as exc:
|
|
251
263
|
if exc.errno == 404:
|
|
@@ -254,6 +266,7 @@ def get_deployment(
|
|
|
254
266
|
else:
|
|
255
267
|
raise
|
|
256
268
|
|
|
269
|
+
# Use workspace group from environment
|
|
257
270
|
if os.environ.get('SINGLESTOREDB_WORKSPACE_GROUP'):
|
|
258
271
|
try:
|
|
259
272
|
return manager.get_workspace_group(
|
|
@@ -267,6 +280,7 @@ def get_deployment(
|
|
|
267
280
|
)
|
|
268
281
|
raise
|
|
269
282
|
|
|
283
|
+
# Use cluster from environment
|
|
270
284
|
if os.environ.get('SINGLESTOREDB_CLUSTER'):
|
|
271
285
|
try:
|
|
272
286
|
return manager.get_starter_workspace(
|
|
@@ -298,11 +312,11 @@ def get_file_space(params: Dict[str, Any]) -> FileSpace:
|
|
|
298
312
|
if file_location:
|
|
299
313
|
file_location_lower_case = file_location.lower()
|
|
300
314
|
|
|
301
|
-
if file_location_lower_case == PERSONAL_SPACE:
|
|
315
|
+
if file_location_lower_case == mgmt_files.PERSONAL_SPACE:
|
|
302
316
|
return manager.personal_space
|
|
303
|
-
elif file_location_lower_case == SHARED_SPACE:
|
|
317
|
+
elif file_location_lower_case == mgmt_files.SHARED_SPACE:
|
|
304
318
|
return manager.shared_space
|
|
305
|
-
elif file_location_lower_case == MODELS_SPACE:
|
|
319
|
+
elif file_location_lower_case == mgmt_files.MODELS_SPACE:
|
|
306
320
|
return manager.models_space
|
|
307
321
|
else:
|
|
308
322
|
raise ValueError(f'invalid file location: {file_location}')
|
|
@@ -10,11 +10,9 @@ import re
|
|
|
10
10
|
from abc import ABC
|
|
11
11
|
from abc import abstractmethod
|
|
12
12
|
from typing import Any
|
|
13
|
-
from typing import BinaryIO
|
|
14
13
|
from typing import Dict
|
|
15
14
|
from typing import List
|
|
16
15
|
from typing import Optional
|
|
17
|
-
from typing import TextIO
|
|
18
16
|
from typing import Union
|
|
19
17
|
|
|
20
18
|
from .. import config
|
|
@@ -350,6 +348,7 @@ class FilesObjectBytesReader(io.BytesIO):
|
|
|
350
348
|
|
|
351
349
|
|
|
352
350
|
class FileLocation(ABC):
|
|
351
|
+
|
|
353
352
|
@abstractmethod
|
|
354
353
|
def open(
|
|
355
354
|
self,
|
|
@@ -362,7 +361,7 @@ class FileLocation(ABC):
|
|
|
362
361
|
@abstractmethod
|
|
363
362
|
def upload_file(
|
|
364
363
|
self,
|
|
365
|
-
local_path: Union[PathLike,
|
|
364
|
+
local_path: Union[PathLike, io.IOBase],
|
|
366
365
|
path: PathLike,
|
|
367
366
|
*,
|
|
368
367
|
overwrite: bool = False,
|
|
@@ -385,7 +384,7 @@ class FileLocation(ABC):
|
|
|
385
384
|
@abstractmethod
|
|
386
385
|
def _upload(
|
|
387
386
|
self,
|
|
388
|
-
content: Union[str, bytes,
|
|
387
|
+
content: Union[str, bytes, io.IOBase],
|
|
389
388
|
path: PathLike,
|
|
390
389
|
*,
|
|
391
390
|
overwrite: bool = False,
|
|
@@ -628,7 +627,7 @@ class FileSpace(FileLocation):
|
|
|
628
627
|
|
|
629
628
|
def upload_file(
|
|
630
629
|
self,
|
|
631
|
-
local_path: Union[PathLike,
|
|
630
|
+
local_path: Union[PathLike, io.IOBase],
|
|
632
631
|
path: PathLike,
|
|
633
632
|
*,
|
|
634
633
|
overwrite: bool = False,
|
|
@@ -646,7 +645,7 @@ class FileSpace(FileLocation):
|
|
|
646
645
|
Should the ``path`` be overwritten if it exists already?
|
|
647
646
|
|
|
648
647
|
"""
|
|
649
|
-
if isinstance(local_path,
|
|
648
|
+
if isinstance(local_path, io.IOBase):
|
|
650
649
|
pass
|
|
651
650
|
elif not os.path.isfile(local_path):
|
|
652
651
|
raise IsADirectoryError(f'local path is not a file: {local_path}')
|
|
@@ -657,8 +656,9 @@ class FileSpace(FileLocation):
|
|
|
657
656
|
|
|
658
657
|
self.remove(path)
|
|
659
658
|
|
|
660
|
-
if isinstance(local_path,
|
|
659
|
+
if isinstance(local_path, io.IOBase):
|
|
661
660
|
return self._upload(local_path, path, overwrite=overwrite)
|
|
661
|
+
|
|
662
662
|
return self._upload(open(local_path, 'rb'), path, overwrite=overwrite)
|
|
663
663
|
|
|
664
664
|
def upload_folder(
|
|
@@ -727,7 +727,7 @@ class FileSpace(FileLocation):
|
|
|
727
727
|
|
|
728
728
|
def _upload(
|
|
729
729
|
self,
|
|
730
|
-
content: Union[str, bytes,
|
|
730
|
+
content: Union[str, bytes, io.IOBase],
|
|
731
731
|
path: PathLike,
|
|
732
732
|
*,
|
|
733
733
|
overwrite: bool = False,
|
|
@@ -10,11 +10,9 @@ import re
|
|
|
10
10
|
import time
|
|
11
11
|
from collections.abc import Mapping
|
|
12
12
|
from typing import Any
|
|
13
|
-
from typing import BinaryIO
|
|
14
13
|
from typing import Dict
|
|
15
14
|
from typing import List
|
|
16
15
|
from typing import Optional
|
|
17
|
-
from typing import TextIO
|
|
18
16
|
from typing import Union
|
|
19
17
|
|
|
20
18
|
from .. import config
|
|
@@ -165,7 +163,7 @@ class Stage(FileLocation):
|
|
|
165
163
|
|
|
166
164
|
def upload_file(
|
|
167
165
|
self,
|
|
168
|
-
local_path: Union[PathLike,
|
|
166
|
+
local_path: Union[PathLike, io.IOBase],
|
|
169
167
|
stage_path: PathLike,
|
|
170
168
|
*,
|
|
171
169
|
overwrite: bool = False,
|
|
@@ -183,7 +181,7 @@ class Stage(FileLocation):
|
|
|
183
181
|
Should the ``stage_path`` be overwritten if it exists already?
|
|
184
182
|
|
|
185
183
|
"""
|
|
186
|
-
if isinstance(local_path,
|
|
184
|
+
if isinstance(local_path, io.IOBase):
|
|
187
185
|
pass
|
|
188
186
|
elif not os.path.isfile(local_path):
|
|
189
187
|
raise IsADirectoryError(f'local path is not a file: {local_path}')
|
|
@@ -194,8 +192,9 @@ class Stage(FileLocation):
|
|
|
194
192
|
|
|
195
193
|
self.remove(stage_path)
|
|
196
194
|
|
|
197
|
-
if isinstance(local_path,
|
|
195
|
+
if isinstance(local_path, io.IOBase):
|
|
198
196
|
return self._upload(local_path, stage_path, overwrite=overwrite)
|
|
197
|
+
|
|
199
198
|
return self._upload(open(local_path, 'rb'), stage_path, overwrite=overwrite)
|
|
200
199
|
|
|
201
200
|
def upload_folder(
|
|
@@ -258,7 +257,7 @@ class Stage(FileLocation):
|
|
|
258
257
|
|
|
259
258
|
def _upload(
|
|
260
259
|
self,
|
|
261
|
-
content: Union[str, bytes,
|
|
260
|
+
content: Union[str, bytes, io.IOBase],
|
|
262
261
|
stage_path: PathLike,
|
|
263
262
|
*,
|
|
264
263
|
overwrite: bool = False,
|
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
import os
|
|
5
5
|
import random
|
|
6
6
|
import secrets
|
|
7
|
+
import tempfile
|
|
7
8
|
import time
|
|
8
9
|
import unittest
|
|
9
10
|
from typing import Any
|
|
@@ -714,3 +715,810 @@ class TestJobsFusion(unittest.TestCase):
|
|
|
714
715
|
res = out[0]
|
|
715
716
|
assert res[0] == job_id
|
|
716
717
|
assert res[1] == 1
|
|
718
|
+
|
|
719
|
+
|
|
720
|
+
@pytest.mark.management
|
|
721
|
+
class TestStageFusion(unittest.TestCase):
|
|
722
|
+
|
|
723
|
+
id: str = secrets.token_hex(8)
|
|
724
|
+
dbname: str = 'information_schema'
|
|
725
|
+
manager: None
|
|
726
|
+
workspace_group: None
|
|
727
|
+
workspace_group_2: None
|
|
728
|
+
|
|
729
|
+
@classmethod
|
|
730
|
+
def setUpClass(cls):
|
|
731
|
+
cls.manager = s2.manage_workspaces()
|
|
732
|
+
us_regions = [x for x in cls.manager.regions if x.name.startswith('US')]
|
|
733
|
+
cls.workspace_group = cls.manager.create_workspace_group(
|
|
734
|
+
f'Stage Fusion Testing 1 {cls.id}',
|
|
735
|
+
region=random.choice(us_regions),
|
|
736
|
+
firewall_ranges=[],
|
|
737
|
+
)
|
|
738
|
+
cls.workspace_group_2 = cls.manager.create_workspace_group(
|
|
739
|
+
f'Stage Fusion Testing 2 {cls.id}',
|
|
740
|
+
region=random.choice(us_regions),
|
|
741
|
+
firewall_ranges=[],
|
|
742
|
+
)
|
|
743
|
+
# Wait for both workspace groups to start
|
|
744
|
+
time.sleep(5)
|
|
745
|
+
|
|
746
|
+
os.environ['SINGLESTOREDB_DEFAULT_DATABASE'] = 'information_schema'
|
|
747
|
+
os.environ['SINGLESTOREDB_WORKSPACE_GROUP'] = cls.workspace_group.id
|
|
748
|
+
|
|
749
|
+
@classmethod
|
|
750
|
+
def tearDownClass(cls):
|
|
751
|
+
if cls.workspace_group is not None:
|
|
752
|
+
cls.workspace_group.terminate(force=True)
|
|
753
|
+
if cls.workspace_group_2 is not None:
|
|
754
|
+
cls.workspace_group_2.terminate(force=True)
|
|
755
|
+
cls.manager = None
|
|
756
|
+
cls.workspace_group = None
|
|
757
|
+
cls.workspace_group_2 = None
|
|
758
|
+
cls.workspace = None
|
|
759
|
+
cls.workspace_2 = None
|
|
760
|
+
if os.environ.get('SINGLESTOREDB_WORKSPACE', None) is not None:
|
|
761
|
+
del os.environ['SINGLESTOREDB_WORKSPACE']
|
|
762
|
+
if os.environ.get('SINGLESTOREDB_WORKSPACE_GROUP', None) is not None:
|
|
763
|
+
del os.environ['SINGLESTOREDB_WORKSPACE_GROUP']
|
|
764
|
+
if os.environ.get('SINGLESTOREDB_DEFAULT_DATABASE', None) is not None:
|
|
765
|
+
del os.environ['SINGLESTOREDB_DEFAULT_DATABASE']
|
|
766
|
+
|
|
767
|
+
def setUp(self):
|
|
768
|
+
self.enabled = os.environ.get('SINGLESTOREDB_FUSION_ENABLED')
|
|
769
|
+
os.environ['SINGLESTOREDB_FUSION_ENABLED'] = '1'
|
|
770
|
+
self.conn = s2.connect(database=type(self).dbname, local_infile=True)
|
|
771
|
+
self.cur = self.conn.cursor()
|
|
772
|
+
|
|
773
|
+
def tearDown(self):
|
|
774
|
+
self._clear_stage()
|
|
775
|
+
|
|
776
|
+
if self.enabled:
|
|
777
|
+
os.environ['SINGLESTOREDB_FUSION_ENABLED'] = self.enabled
|
|
778
|
+
else:
|
|
779
|
+
del os.environ['SINGLESTOREDB_FUSION_ENABLED']
|
|
780
|
+
|
|
781
|
+
try:
|
|
782
|
+
if self.cur is not None:
|
|
783
|
+
self.cur.close()
|
|
784
|
+
except Exception:
|
|
785
|
+
# traceback.print_exc()
|
|
786
|
+
pass
|
|
787
|
+
|
|
788
|
+
try:
|
|
789
|
+
if self.conn is not None:
|
|
790
|
+
self.conn.close()
|
|
791
|
+
except Exception:
|
|
792
|
+
# traceback.print_exc()
|
|
793
|
+
pass
|
|
794
|
+
|
|
795
|
+
def _clear_stage(self):
|
|
796
|
+
if self.workspace_group is not None:
|
|
797
|
+
self.cur.execute(f'''
|
|
798
|
+
show stage files
|
|
799
|
+
in group id '{self.workspace_group.id}' recursive
|
|
800
|
+
''')
|
|
801
|
+
files = list(self.cur)
|
|
802
|
+
folders = []
|
|
803
|
+
for file in files:
|
|
804
|
+
if file[0].endswith('/'):
|
|
805
|
+
folders.append(file)
|
|
806
|
+
continue
|
|
807
|
+
self.cur.execute(f'''
|
|
808
|
+
drop stage file '{file[0]}'
|
|
809
|
+
in group id '{self.workspace_group.id}'
|
|
810
|
+
''')
|
|
811
|
+
for folder in folders:
|
|
812
|
+
self.cur.execute(f'''
|
|
813
|
+
drop stage folder '{folder[0]}'
|
|
814
|
+
in group id '{self.workspace_group.id}'
|
|
815
|
+
''')
|
|
816
|
+
|
|
817
|
+
if self.workspace_group_2 is not None:
|
|
818
|
+
self.cur.execute(f'''
|
|
819
|
+
show stage files
|
|
820
|
+
in group id '{self.workspace_group_2.id}' recursive
|
|
821
|
+
''')
|
|
822
|
+
files = list(self.cur)
|
|
823
|
+
folders = []
|
|
824
|
+
for file in files:
|
|
825
|
+
if file[0].endswith('/'):
|
|
826
|
+
folders.append(file)
|
|
827
|
+
continue
|
|
828
|
+
self.cur.execute(f'''
|
|
829
|
+
drop stage file '{file[0]}'
|
|
830
|
+
in group id '{self.workspace_group_2.id}'
|
|
831
|
+
''')
|
|
832
|
+
for folder in folders:
|
|
833
|
+
self.cur.execute(f'''
|
|
834
|
+
drop stage folder '{folder[0]}'
|
|
835
|
+
in group id '{self.workspace_group_2.id}'
|
|
836
|
+
''')
|
|
837
|
+
|
|
838
|
+
def test_show_stage(self):
|
|
839
|
+
test2_sql = os.path.join(os.path.dirname(__file__), 'test2.sql')
|
|
840
|
+
|
|
841
|
+
# Should be empty
|
|
842
|
+
self.cur.execute('''
|
|
843
|
+
show stage files
|
|
844
|
+
''')
|
|
845
|
+
files = list(self.cur)
|
|
846
|
+
assert len(files) == 0
|
|
847
|
+
|
|
848
|
+
# Copy files to stage
|
|
849
|
+
self.cur.execute(
|
|
850
|
+
f'upload file to stage "new_test_1.sql" from "{test2_sql}"',
|
|
851
|
+
)
|
|
852
|
+
self.cur.execute('create stage folder "subdir1"')
|
|
853
|
+
self.cur.execute(
|
|
854
|
+
f'upload file to stage "subdir1/new_test_2.sql" from "{test2_sql}"',
|
|
855
|
+
)
|
|
856
|
+
self.cur.execute(
|
|
857
|
+
f'upload file to stage "subdir1/new_test_3.sql" from "{test2_sql}"',
|
|
858
|
+
)
|
|
859
|
+
self.cur.execute('create stage folder "subdir2"')
|
|
860
|
+
self.cur.execute(
|
|
861
|
+
f'upload file to stage "subdir2/new_test_4.sql" from "{test2_sql}"',
|
|
862
|
+
)
|
|
863
|
+
self.cur.execute(
|
|
864
|
+
f'upload file to stage "subdir2/new_test_5.sql" from "{test2_sql}"',
|
|
865
|
+
)
|
|
866
|
+
|
|
867
|
+
# Make sure files are there
|
|
868
|
+
self.cur.execute('''
|
|
869
|
+
show stage files recursive
|
|
870
|
+
''')
|
|
871
|
+
files = list(self.cur)
|
|
872
|
+
assert len(files) == 7
|
|
873
|
+
assert list(sorted(x[0] for x in files)) == [
|
|
874
|
+
'new_test_1.sql',
|
|
875
|
+
'subdir1/',
|
|
876
|
+
'subdir1/new_test_2.sql',
|
|
877
|
+
'subdir1/new_test_3.sql',
|
|
878
|
+
'subdir2/',
|
|
879
|
+
'subdir2/new_test_4.sql',
|
|
880
|
+
'subdir2/new_test_5.sql',
|
|
881
|
+
]
|
|
882
|
+
|
|
883
|
+
# Do non-recursive listing
|
|
884
|
+
self.cur.execute('''
|
|
885
|
+
show stage files
|
|
886
|
+
''')
|
|
887
|
+
files = list(self.cur)
|
|
888
|
+
assert len(files) == 3
|
|
889
|
+
assert list(sorted(x[0] for x in files)) == [
|
|
890
|
+
'new_test_1.sql',
|
|
891
|
+
'subdir1/',
|
|
892
|
+
'subdir2/',
|
|
893
|
+
]
|
|
894
|
+
|
|
895
|
+
# List files in specific workspace group
|
|
896
|
+
self.cur.execute(f'''
|
|
897
|
+
show stage files in group id '{self.workspace_group.id}'
|
|
898
|
+
''')
|
|
899
|
+
files = list(self.cur)
|
|
900
|
+
assert len(files) == 3
|
|
901
|
+
assert list(sorted(x[0] for x in files)) == [
|
|
902
|
+
'new_test_1.sql',
|
|
903
|
+
'subdir1/',
|
|
904
|
+
'subdir2/',
|
|
905
|
+
]
|
|
906
|
+
|
|
907
|
+
self.cur.execute(f'''
|
|
908
|
+
show stage files in id '{self.workspace_group.id}'
|
|
909
|
+
''')
|
|
910
|
+
files = list(self.cur)
|
|
911
|
+
assert len(files) == 3
|
|
912
|
+
assert list(sorted(x[0] for x in files)) == [
|
|
913
|
+
'new_test_1.sql',
|
|
914
|
+
'subdir1/',
|
|
915
|
+
'subdir2/',
|
|
916
|
+
]
|
|
917
|
+
|
|
918
|
+
self.cur.execute(f'''
|
|
919
|
+
show stage files in group '{self.workspace_group.name}'
|
|
920
|
+
''')
|
|
921
|
+
files = list(self.cur)
|
|
922
|
+
assert len(files) == 3
|
|
923
|
+
assert list(sorted(x[0] for x in files)) == [
|
|
924
|
+
'new_test_1.sql',
|
|
925
|
+
'subdir1/',
|
|
926
|
+
'subdir2/',
|
|
927
|
+
]
|
|
928
|
+
|
|
929
|
+
self.cur.execute(f'''
|
|
930
|
+
show stage files in '{self.workspace_group.name}'
|
|
931
|
+
''')
|
|
932
|
+
files = list(self.cur)
|
|
933
|
+
assert len(files) == 3
|
|
934
|
+
assert list(sorted(x[0] for x in files)) == [
|
|
935
|
+
'new_test_1.sql',
|
|
936
|
+
'subdir1/',
|
|
937
|
+
'subdir2/',
|
|
938
|
+
]
|
|
939
|
+
|
|
940
|
+
# Check other workspace group
|
|
941
|
+
self.cur.execute(f'''
|
|
942
|
+
show stage files in group '{self.workspace_group_2.name}'
|
|
943
|
+
''')
|
|
944
|
+
files = list(self.cur)
|
|
945
|
+
assert len(files) == 0
|
|
946
|
+
|
|
947
|
+
# Limit results
|
|
948
|
+
self.cur.execute('''
|
|
949
|
+
show stage files recursive limit 5
|
|
950
|
+
''')
|
|
951
|
+
files = list(self.cur)
|
|
952
|
+
assert len(files) == 5
|
|
953
|
+
assert list(sorted(x[0] for x in files)) == [
|
|
954
|
+
'new_test_1.sql',
|
|
955
|
+
'subdir1/',
|
|
956
|
+
'subdir1/new_test_2.sql',
|
|
957
|
+
'subdir1/new_test_3.sql',
|
|
958
|
+
'subdir2/',
|
|
959
|
+
]
|
|
960
|
+
|
|
961
|
+
# Order by type and name
|
|
962
|
+
self.cur.execute('''
|
|
963
|
+
show stage files order by type, name recursive extended
|
|
964
|
+
''')
|
|
965
|
+
files = list(self.cur)
|
|
966
|
+
assert len(files) == 7
|
|
967
|
+
assert list(x[0] for x in files) == [
|
|
968
|
+
'subdir1/',
|
|
969
|
+
'subdir2/',
|
|
970
|
+
'new_test_1.sql',
|
|
971
|
+
'subdir1/new_test_2.sql',
|
|
972
|
+
'subdir1/new_test_3.sql',
|
|
973
|
+
'subdir2/new_test_4.sql',
|
|
974
|
+
'subdir2/new_test_5.sql',
|
|
975
|
+
]
|
|
976
|
+
|
|
977
|
+
# Order by type and name descending
|
|
978
|
+
self.cur.execute('''
|
|
979
|
+
show stage files order by type desc, name desc recursive extended
|
|
980
|
+
''')
|
|
981
|
+
files = list(self.cur)
|
|
982
|
+
assert len(files) == 7
|
|
983
|
+
assert list(x[0] for x in files) == [
|
|
984
|
+
'subdir2/new_test_5.sql',
|
|
985
|
+
'subdir2/new_test_4.sql',
|
|
986
|
+
'subdir1/new_test_3.sql',
|
|
987
|
+
'subdir1/new_test_2.sql',
|
|
988
|
+
'new_test_1.sql',
|
|
989
|
+
'subdir2/',
|
|
990
|
+
'subdir1/',
|
|
991
|
+
]
|
|
992
|
+
|
|
993
|
+
# List at specific path
|
|
994
|
+
self.cur.execute('''
|
|
995
|
+
show stage files at 'subdir2/' recursive
|
|
996
|
+
''')
|
|
997
|
+
files = list(self.cur)
|
|
998
|
+
assert len(files) == 2
|
|
999
|
+
assert list(sorted(x[0] for x in files)) == [
|
|
1000
|
+
'new_test_4.sql',
|
|
1001
|
+
'new_test_5.sql',
|
|
1002
|
+
]
|
|
1003
|
+
|
|
1004
|
+
# LIKE clause
|
|
1005
|
+
self.cur.execute('''
|
|
1006
|
+
show stage files like '%_4.%' recursive
|
|
1007
|
+
''')
|
|
1008
|
+
files = list(self.cur)
|
|
1009
|
+
assert len(files) == 1
|
|
1010
|
+
assert list(sorted(x[0] for x in files)) == [
|
|
1011
|
+
'subdir2/new_test_4.sql',
|
|
1012
|
+
]
|
|
1013
|
+
|
|
1014
|
+
def test_download_stage(self):
|
|
1015
|
+
test2_sql = os.path.join(os.path.dirname(__file__), 'test2.sql')
|
|
1016
|
+
|
|
1017
|
+
# Should be empty
|
|
1018
|
+
self.cur.execute('''
|
|
1019
|
+
show stage files
|
|
1020
|
+
''')
|
|
1021
|
+
files = list(self.cur)
|
|
1022
|
+
assert len(files) == 0
|
|
1023
|
+
|
|
1024
|
+
# Copy file to stage 1
|
|
1025
|
+
self.cur.execute(f'''
|
|
1026
|
+
upload file to stage 'dl_test.sql' from '{test2_sql}'
|
|
1027
|
+
''')
|
|
1028
|
+
|
|
1029
|
+
self.cur.execute('''
|
|
1030
|
+
show stage files
|
|
1031
|
+
''')
|
|
1032
|
+
files = list(self.cur)
|
|
1033
|
+
assert len(files) == 1
|
|
1034
|
+
assert list(sorted(x[0] for x in files)) == ['dl_test.sql']
|
|
1035
|
+
|
|
1036
|
+
# Copy file to stage 2
|
|
1037
|
+
self.cur.execute(f'''
|
|
1038
|
+
upload file to stage 'dl_test2.sql'
|
|
1039
|
+
in group '{self.workspace_group_2.name}'
|
|
1040
|
+
from '{test2_sql}'
|
|
1041
|
+
''')
|
|
1042
|
+
|
|
1043
|
+
# Make sure only one file in stage 2
|
|
1044
|
+
self.cur.execute(f'''
|
|
1045
|
+
show stage files in group '{self.workspace_group_2.name}'
|
|
1046
|
+
''')
|
|
1047
|
+
files = list(self.cur)
|
|
1048
|
+
assert len(files) == 1
|
|
1049
|
+
assert list(sorted(x[0] for x in files)) == ['dl_test2.sql']
|
|
1050
|
+
|
|
1051
|
+
# Download file from stage 1
|
|
1052
|
+
with tempfile.TemporaryDirectory() as tmpdir:
|
|
1053
|
+
self.cur.execute(f'''
|
|
1054
|
+
download stage file 'dl_test.sql' to '{tmpdir}/dl_test.sql'
|
|
1055
|
+
''')
|
|
1056
|
+
with open(os.path.join(tmpdir, 'dl_test.sql'), 'r') as dl_file:
|
|
1057
|
+
assert dl_file.read() == open(test2_sql, 'r').read()
|
|
1058
|
+
|
|
1059
|
+
# Download file from stage 2
|
|
1060
|
+
with tempfile.TemporaryDirectory() as tmpdir:
|
|
1061
|
+
self.cur.execute(f'''
|
|
1062
|
+
download stage file 'dl_test2.sql'
|
|
1063
|
+
in group '{self.workspace_group_2.name}'
|
|
1064
|
+
to '{tmpdir}/dl_test2.sql'
|
|
1065
|
+
''')
|
|
1066
|
+
with open(os.path.join(tmpdir, 'dl_test2.sql'), 'r') as dl_file:
|
|
1067
|
+
assert dl_file.read() == open(test2_sql, 'r').read()
|
|
1068
|
+
|
|
1069
|
+
def test_stage_multi_wg_operations(self):
|
|
1070
|
+
test_sql = os.path.join(os.path.dirname(__file__), 'test.sql')
|
|
1071
|
+
test2_sql = os.path.join(os.path.dirname(__file__), 'test2.sql')
|
|
1072
|
+
|
|
1073
|
+
# Should be empty
|
|
1074
|
+
self.cur.execute('''
|
|
1075
|
+
show stage files
|
|
1076
|
+
''')
|
|
1077
|
+
files = list(self.cur)
|
|
1078
|
+
assert len(files) == 0
|
|
1079
|
+
|
|
1080
|
+
# Copy file to stage 1
|
|
1081
|
+
self.cur.execute(f'''
|
|
1082
|
+
upload file to stage 'new_test.sql' from '{test_sql}'
|
|
1083
|
+
''')
|
|
1084
|
+
|
|
1085
|
+
self.cur.execute('''
|
|
1086
|
+
show stage files
|
|
1087
|
+
''')
|
|
1088
|
+
files = list(self.cur)
|
|
1089
|
+
assert len(files) == 1
|
|
1090
|
+
|
|
1091
|
+
# Copy file to stage 2
|
|
1092
|
+
self.cur.execute(f'''
|
|
1093
|
+
upload file to stage 'new_test2.sql'
|
|
1094
|
+
in group '{self.workspace_group_2.name}'
|
|
1095
|
+
from '{test2_sql}'
|
|
1096
|
+
''')
|
|
1097
|
+
|
|
1098
|
+
# Make sure only one file in stage 1
|
|
1099
|
+
self.cur.execute('''
|
|
1100
|
+
show stage files
|
|
1101
|
+
''')
|
|
1102
|
+
files = list(self.cur)
|
|
1103
|
+
assert len(files) == 1
|
|
1104
|
+
assert files[0][0] == 'new_test.sql'
|
|
1105
|
+
|
|
1106
|
+
# Make sure only one file in stage 2
|
|
1107
|
+
self.cur.execute(f'''
|
|
1108
|
+
show stage files in group '{self.workspace_group_2.name}' recursive
|
|
1109
|
+
''')
|
|
1110
|
+
files = list(self.cur)
|
|
1111
|
+
assert len(files) == 1
|
|
1112
|
+
assert list(sorted(x[0] for x in files)) == ['new_test2.sql']
|
|
1113
|
+
|
|
1114
|
+
# Make sure only one file in stage 2 (using IN)
|
|
1115
|
+
self.cur.execute(f'''
|
|
1116
|
+
show stage files in '{self.workspace_group_2.name}' recursive
|
|
1117
|
+
''')
|
|
1118
|
+
files = list(self.cur)
|
|
1119
|
+
assert len(files) == 1
|
|
1120
|
+
assert list(sorted(x[0] for x in files)) == ['new_test2.sql']
|
|
1121
|
+
|
|
1122
|
+
# Make subdir
|
|
1123
|
+
self.cur.execute(f'''
|
|
1124
|
+
create stage folder 'data' in group '{self.workspace_group_2.name}'
|
|
1125
|
+
''')
|
|
1126
|
+
|
|
1127
|
+
# Upload file using workspace ID
|
|
1128
|
+
self.cur.execute(f'''
|
|
1129
|
+
upload file to stage 'data/new_test2_sub.sql'
|
|
1130
|
+
in group id '{self.workspace_group_2.id}'
|
|
1131
|
+
from '{test2_sql}'
|
|
1132
|
+
''')
|
|
1133
|
+
|
|
1134
|
+
# Make sure only one file in stage 1
|
|
1135
|
+
self.cur.execute('''
|
|
1136
|
+
show stage files
|
|
1137
|
+
''')
|
|
1138
|
+
files = list(self.cur)
|
|
1139
|
+
assert len(files) == 1
|
|
1140
|
+
assert files[0][0] == 'new_test.sql'
|
|
1141
|
+
|
|
1142
|
+
# Make sure two files in stage 2
|
|
1143
|
+
self.cur.execute(f'''
|
|
1144
|
+
show stage files in group id '{self.workspace_group_2.id}' recursive
|
|
1145
|
+
''')
|
|
1146
|
+
files = list(self.cur)
|
|
1147
|
+
assert len(files) == 3
|
|
1148
|
+
assert list(sorted(x[0] for x in files)) == \
|
|
1149
|
+
['data/', 'data/new_test2_sub.sql', 'new_test2.sql']
|
|
1150
|
+
|
|
1151
|
+
# Test overwrite
|
|
1152
|
+
with self.assertRaises(OSError):
|
|
1153
|
+
self.cur.execute(f'''
|
|
1154
|
+
upload file to stage 'data/new_test2_sub.sql'
|
|
1155
|
+
in group id '{self.workspace_group_2.id}'
|
|
1156
|
+
from '{test2_sql}'
|
|
1157
|
+
''')
|
|
1158
|
+
|
|
1159
|
+
self.cur.execute(f'''
|
|
1160
|
+
upload file to stage 'data/new_test2_sub.sql'
|
|
1161
|
+
in group id '{self.workspace_group_2.id}'
|
|
1162
|
+
from '{test2_sql}' overwrite
|
|
1163
|
+
''')
|
|
1164
|
+
|
|
1165
|
+
# Make sure two files in stage 2
|
|
1166
|
+
self.cur.execute(f'''
|
|
1167
|
+
show stage files in group id '{self.workspace_group_2.id}' recursive
|
|
1168
|
+
''')
|
|
1169
|
+
files = list(self.cur)
|
|
1170
|
+
assert len(files) == 3
|
|
1171
|
+
assert list(sorted(x[0] for x in files)) == \
|
|
1172
|
+
['data/', 'data/new_test2_sub.sql', 'new_test2.sql']
|
|
1173
|
+
|
|
1174
|
+
# Test LIKE clause
|
|
1175
|
+
self.cur.execute(f'''
|
|
1176
|
+
show stage files
|
|
1177
|
+
in group id '{self.workspace_group_2.id}'
|
|
1178
|
+
like '%_sub%' recursive
|
|
1179
|
+
''')
|
|
1180
|
+
files = list(self.cur)
|
|
1181
|
+
assert len(files) == 1
|
|
1182
|
+
assert list(sorted(x[0] for x in files)) == ['data/new_test2_sub.sql']
|
|
1183
|
+
|
|
1184
|
+
# Drop file from default stage
|
|
1185
|
+
self.cur.execute('''
|
|
1186
|
+
drop stage file 'new_test.sql'
|
|
1187
|
+
''')
|
|
1188
|
+
|
|
1189
|
+
# Make sure no files in stage 1
|
|
1190
|
+
self.cur.execute('''
|
|
1191
|
+
show stage files
|
|
1192
|
+
''')
|
|
1193
|
+
files = list(self.cur)
|
|
1194
|
+
assert len(files) == 0
|
|
1195
|
+
|
|
1196
|
+
# Make sure two files in stage 2
|
|
1197
|
+
self.cur.execute(f'''
|
|
1198
|
+
show stage files in group id '{self.workspace_group_2.id}' recursive
|
|
1199
|
+
''')
|
|
1200
|
+
files = list(self.cur)
|
|
1201
|
+
assert len(files) == 3
|
|
1202
|
+
assert list(sorted(x[0] for x in files)) == \
|
|
1203
|
+
['data/', 'data/new_test2_sub.sql', 'new_test2.sql']
|
|
1204
|
+
|
|
1205
|
+
# Attempt to drop directory from stage 2
|
|
1206
|
+
with self.assertRaises(OSError):
|
|
1207
|
+
self.cur.execute(f'''
|
|
1208
|
+
drop stage folder 'data'
|
|
1209
|
+
in group id '{self.workspace_group_2.id}'
|
|
1210
|
+
''')
|
|
1211
|
+
|
|
1212
|
+
self.cur.execute(f'''
|
|
1213
|
+
drop stage file 'data/new_test2_sub.sql'
|
|
1214
|
+
in group id '{self.workspace_group_2.id}'
|
|
1215
|
+
''')
|
|
1216
|
+
|
|
1217
|
+
# Make sure one file and one directory in stage 2
|
|
1218
|
+
self.cur.execute(f'''
|
|
1219
|
+
show stage files in group id '{self.workspace_group_2.id}' recursive
|
|
1220
|
+
''')
|
|
1221
|
+
files = list(self.cur)
|
|
1222
|
+
assert len(files) == 2
|
|
1223
|
+
assert list(sorted(x[0] for x in files)) == ['data/', 'new_test2.sql']
|
|
1224
|
+
|
|
1225
|
+
# Drop stage folder from stage 2
|
|
1226
|
+
self.cur.execute(f'''
|
|
1227
|
+
drop stage folder 'data'
|
|
1228
|
+
in group id '{self.workspace_group_2.id}'
|
|
1229
|
+
''')
|
|
1230
|
+
|
|
1231
|
+
# Make sure one file in stage 2
|
|
1232
|
+
self.cur.execute(f'''
|
|
1233
|
+
show stage files in group id '{self.workspace_group_2.id}' recursive
|
|
1234
|
+
''')
|
|
1235
|
+
files = list(self.cur)
|
|
1236
|
+
assert len(files) == 1
|
|
1237
|
+
assert list(sorted(x[0] for x in files)) == ['new_test2.sql']
|
|
1238
|
+
|
|
1239
|
+
# Drop last file
|
|
1240
|
+
self.cur.execute(f'''
|
|
1241
|
+
drop stage file 'new_test2.sql'
|
|
1242
|
+
in group id '{self.workspace_group_2.id}'
|
|
1243
|
+
''')
|
|
1244
|
+
|
|
1245
|
+
# Make sure no files in stage 2
|
|
1246
|
+
self.cur.execute(f'''
|
|
1247
|
+
show stage files in group id '{self.workspace_group_2.id}' recursive
|
|
1248
|
+
''')
|
|
1249
|
+
files = list(self.cur)
|
|
1250
|
+
assert len(files) == 0
|
|
1251
|
+
|
|
1252
|
+
|
|
1253
|
+
@pytest.mark.management
|
|
1254
|
+
class TestFilesFusion(unittest.TestCase):
|
|
1255
|
+
|
|
1256
|
+
id: str = secrets.token_hex(8)
|
|
1257
|
+
dbname: str = 'information_schema'
|
|
1258
|
+
manager: None
|
|
1259
|
+
workspace_group: None
|
|
1260
|
+
|
|
1261
|
+
@classmethod
|
|
1262
|
+
def setUpClass(cls):
|
|
1263
|
+
cls.manager = s2.manage_workspaces()
|
|
1264
|
+
us_regions = [x for x in cls.manager.regions if x.name.startswith('US')]
|
|
1265
|
+
cls.workspace_group = cls.manager.create_workspace_group(
|
|
1266
|
+
f'Files Fusion Testing {cls.id}',
|
|
1267
|
+
region=random.choice(us_regions),
|
|
1268
|
+
firewall_ranges=[],
|
|
1269
|
+
)
|
|
1270
|
+
# Wait for both workspace groups to start
|
|
1271
|
+
time.sleep(5)
|
|
1272
|
+
|
|
1273
|
+
os.environ['SINGLESTOREDB_DEFAULT_DATABASE'] = 'information_schema'
|
|
1274
|
+
os.environ['SINGLESTOREDB_WORKSPACE_GROUP'] = cls.workspace_group.id
|
|
1275
|
+
|
|
1276
|
+
@classmethod
|
|
1277
|
+
def tearDownClass(cls):
|
|
1278
|
+
if cls.workspace_group is not None:
|
|
1279
|
+
cls.workspace_group.terminate(force=True)
|
|
1280
|
+
cls.manager = None
|
|
1281
|
+
cls.workspace_group = None
|
|
1282
|
+
cls.workspace = None
|
|
1283
|
+
if os.environ.get('SINGLESTOREDB_WORKSPACE', None) is not None:
|
|
1284
|
+
del os.environ['SINGLESTOREDB_WORKSPACE']
|
|
1285
|
+
if os.environ.get('SINGLESTOREDB_WORKSPACE_GROUP', None) is not None:
|
|
1286
|
+
del os.environ['SINGLESTOREDB_WORKSPACE_GROUP']
|
|
1287
|
+
if os.environ.get('SINGLESTOREDB_DEFAULT_DATABASE', None) is not None:
|
|
1288
|
+
del os.environ['SINGLESTOREDB_DEFAULT_DATABASE']
|
|
1289
|
+
|
|
1290
|
+
def setUp(self):
|
|
1291
|
+
self.enabled = os.environ.get('SINGLESTOREDB_FUSION_ENABLED')
|
|
1292
|
+
os.environ['SINGLESTOREDB_FUSION_ENABLED'] = '1'
|
|
1293
|
+
self.conn = s2.connect(database=type(self).dbname, local_infile=True)
|
|
1294
|
+
self.cur = self.conn.cursor()
|
|
1295
|
+
|
|
1296
|
+
def tearDown(self):
|
|
1297
|
+
self._clear_files()
|
|
1298
|
+
|
|
1299
|
+
if self.enabled:
|
|
1300
|
+
os.environ['SINGLESTOREDB_FUSION_ENABLED'] = self.enabled
|
|
1301
|
+
else:
|
|
1302
|
+
del os.environ['SINGLESTOREDB_FUSION_ENABLED']
|
|
1303
|
+
|
|
1304
|
+
try:
|
|
1305
|
+
if self.cur is not None:
|
|
1306
|
+
self.cur.close()
|
|
1307
|
+
except Exception:
|
|
1308
|
+
# traceback.print_exc()
|
|
1309
|
+
pass
|
|
1310
|
+
|
|
1311
|
+
try:
|
|
1312
|
+
if self.conn is not None:
|
|
1313
|
+
self.conn.close()
|
|
1314
|
+
except Exception:
|
|
1315
|
+
# traceback.print_exc()
|
|
1316
|
+
pass
|
|
1317
|
+
|
|
1318
|
+
def _clear_files(self):
|
|
1319
|
+
cls = type(self)
|
|
1320
|
+
for prefix in ['show', 'dl', 'drop']:
|
|
1321
|
+
for i in range(1, 6):
|
|
1322
|
+
try:
|
|
1323
|
+
self.cur.execute(
|
|
1324
|
+
f'''drop personal file "{prefix}_test_{i}_{cls.id}.ipynb"''',
|
|
1325
|
+
)
|
|
1326
|
+
except (OSError, s2.ManagementError):
|
|
1327
|
+
pass
|
|
1328
|
+
for i in range(1, 6):
|
|
1329
|
+
try:
|
|
1330
|
+
self.cur.execute(
|
|
1331
|
+
f'''drop shared file "{prefix}_test_{i}_{cls.id}.ipynb"''',
|
|
1332
|
+
)
|
|
1333
|
+
except (OSError, s2.ManagementError):
|
|
1334
|
+
pass
|
|
1335
|
+
|
|
1336
|
+
def test_show_personal_files(self):
|
|
1337
|
+
return self._test_show_files('personal')
|
|
1338
|
+
|
|
1339
|
+
def test_show_shared_files(self):
|
|
1340
|
+
return self._test_show_files('shared')
|
|
1341
|
+
|
|
1342
|
+
def _test_show_files(self, ftype):
|
|
1343
|
+
cls = type(self)
|
|
1344
|
+
nb = os.path.join(os.path.dirname(__file__), 'test.ipynb')
|
|
1345
|
+
|
|
1346
|
+
# Should be empty
|
|
1347
|
+
self.cur.execute(f'''
|
|
1348
|
+
show {ftype} files like 'show_%{cls.id}%'
|
|
1349
|
+
''')
|
|
1350
|
+
files = list(self.cur)
|
|
1351
|
+
assert len(files) == 0
|
|
1352
|
+
|
|
1353
|
+
# Upload files
|
|
1354
|
+
self.cur.execute(
|
|
1355
|
+
f'upload {ftype} file to "show_test_1_{cls.id}.ipynb" from "{nb}"',
|
|
1356
|
+
)
|
|
1357
|
+
self.cur.execute(
|
|
1358
|
+
f'upload {ftype} file to "show_test_2_{cls.id}.ipynb" from "{nb}"',
|
|
1359
|
+
)
|
|
1360
|
+
self.cur.execute(
|
|
1361
|
+
f'upload {ftype} file to "show_test_3_{cls.id}.ipynb" from "{nb}"',
|
|
1362
|
+
)
|
|
1363
|
+
self.cur.execute(
|
|
1364
|
+
f'upload {ftype} file to "show_test_4_{cls.id}.ipynb" from "{nb}"',
|
|
1365
|
+
)
|
|
1366
|
+
self.cur.execute(
|
|
1367
|
+
f'upload {ftype} file to "show_test_5_{cls.id}.ipynb" from "{nb}"',
|
|
1368
|
+
)
|
|
1369
|
+
|
|
1370
|
+
# Make sure files are there
|
|
1371
|
+
self.cur.execute(f'''
|
|
1372
|
+
show {ftype} files like 'show_%{cls.id}%'
|
|
1373
|
+
''')
|
|
1374
|
+
files = list(self.cur)
|
|
1375
|
+
assert len(files) == 5
|
|
1376
|
+
assert list(sorted(x[0] for x in files)) == [
|
|
1377
|
+
f'show_test_1_{cls.id}.ipynb',
|
|
1378
|
+
f'show_test_2_{cls.id}.ipynb',
|
|
1379
|
+
f'show_test_3_{cls.id}.ipynb',
|
|
1380
|
+
f'show_test_4_{cls.id}.ipynb',
|
|
1381
|
+
f'show_test_5_{cls.id}.ipynb',
|
|
1382
|
+
]
|
|
1383
|
+
|
|
1384
|
+
# Test ORDER BY
|
|
1385
|
+
self.cur.execute(f'''
|
|
1386
|
+
show {ftype} files like 'show_%{cls.id}%' order by name desc
|
|
1387
|
+
''')
|
|
1388
|
+
files = list(self.cur)
|
|
1389
|
+
assert len(files) == 5
|
|
1390
|
+
assert list(x[0] for x in files) == [
|
|
1391
|
+
f'show_test_5_{cls.id}.ipynb',
|
|
1392
|
+
f'show_test_4_{cls.id}.ipynb',
|
|
1393
|
+
f'show_test_3_{cls.id}.ipynb',
|
|
1394
|
+
f'show_test_2_{cls.id}.ipynb',
|
|
1395
|
+
f'show_test_1_{cls.id}.ipynb',
|
|
1396
|
+
]
|
|
1397
|
+
|
|
1398
|
+
# Test LIMIT
|
|
1399
|
+
self.cur.execute(f'''
|
|
1400
|
+
show {ftype} files like 'show_%{cls.id}%' order by name desc limit 3
|
|
1401
|
+
''')
|
|
1402
|
+
files = list(self.cur)
|
|
1403
|
+
assert len(files) == 3
|
|
1404
|
+
assert list(x[0] for x in files) == [
|
|
1405
|
+
f'show_test_5_{cls.id}.ipynb',
|
|
1406
|
+
f'show_test_4_{cls.id}.ipynb',
|
|
1407
|
+
f'show_test_3_{cls.id}.ipynb',
|
|
1408
|
+
]
|
|
1409
|
+
|
|
1410
|
+
# Test EXTENDED
|
|
1411
|
+
self.cur.execute(f'''
|
|
1412
|
+
show {ftype} files like 'show_%{cls.id}%' extended
|
|
1413
|
+
''')
|
|
1414
|
+
assert [x[0] for x in self.cur.description] == \
|
|
1415
|
+
['Name', 'Type', 'Size', 'Writable', 'CreatedAt', 'LastModifiedAt']
|
|
1416
|
+
|
|
1417
|
+
def test_download_personal_files(self):
|
|
1418
|
+
return self._test_download_files('personal')
|
|
1419
|
+
|
|
1420
|
+
def test_download_shared_files(self):
|
|
1421
|
+
return self._test_download_files('shared')
|
|
1422
|
+
|
|
1423
|
+
def _test_download_files(self, ftype):
|
|
1424
|
+
cls = type(self)
|
|
1425
|
+
nb = os.path.join(os.path.dirname(__file__), 'test.ipynb')
|
|
1426
|
+
|
|
1427
|
+
# Should be empty
|
|
1428
|
+
self.cur.execute(f'''
|
|
1429
|
+
show {ftype} files like 'dl_%{cls.id}%'
|
|
1430
|
+
''')
|
|
1431
|
+
files = list(self.cur)
|
|
1432
|
+
assert len(files) == 0
|
|
1433
|
+
|
|
1434
|
+
# Upload files
|
|
1435
|
+
self.cur.execute(f'upload {ftype} file to "dl_test_1_{cls.id}.ipynb" from "{nb}"')
|
|
1436
|
+
self.cur.execute(f'upload {ftype} file to "dl_test_2_{cls.id}.ipynb" from "{nb}"')
|
|
1437
|
+
|
|
1438
|
+
# Make sure files are there
|
|
1439
|
+
self.cur.execute(f'''
|
|
1440
|
+
show {ftype} files like 'dl_%{cls.id}%'
|
|
1441
|
+
''')
|
|
1442
|
+
files = list(self.cur)
|
|
1443
|
+
assert len(files) == 2
|
|
1444
|
+
assert list(sorted(x[0] for x in files)) == [
|
|
1445
|
+
f'dl_test_1_{cls.id}.ipynb',
|
|
1446
|
+
f'dl_test_2_{cls.id}.ipynb',
|
|
1447
|
+
]
|
|
1448
|
+
|
|
1449
|
+
# Download files
|
|
1450
|
+
with tempfile.TemporaryDirectory() as tmpdir:
|
|
1451
|
+
self.cur.execute(f'''
|
|
1452
|
+
download {ftype} file 'dl_test_1_{cls.id}.ipynb'
|
|
1453
|
+
to '{tmpdir}/dl_test_1.ipynb'
|
|
1454
|
+
''')
|
|
1455
|
+
with open(os.path.join(tmpdir, 'dl_test_1.ipynb'), 'r') as dl_file:
|
|
1456
|
+
assert dl_file.read() == open(nb, 'r').read()
|
|
1457
|
+
|
|
1458
|
+
self.cur.execute(f'''
|
|
1459
|
+
download {ftype} file 'dl_test_2_{cls.id}.ipynb'
|
|
1460
|
+
to '{tmpdir}/dl_test_2.ipynb'
|
|
1461
|
+
''')
|
|
1462
|
+
with open(os.path.join(tmpdir, 'dl_test_2.ipynb'), 'r') as dl_file:
|
|
1463
|
+
assert dl_file.read() == open(nb, 'r').read()
|
|
1464
|
+
|
|
1465
|
+
def test_drop_personal_files(self):
|
|
1466
|
+
return self._test_drop_files('personal')
|
|
1467
|
+
|
|
1468
|
+
def test_drop_shared_files(self):
|
|
1469
|
+
return self._test_drop_files('shared')
|
|
1470
|
+
|
|
1471
|
+
def _test_drop_files(self, ftype):
|
|
1472
|
+
cls = type(self)
|
|
1473
|
+
nb = os.path.join(os.path.dirname(__file__), 'test.ipynb')
|
|
1474
|
+
|
|
1475
|
+
# Should be empty
|
|
1476
|
+
self.cur.execute(f'''
|
|
1477
|
+
show {ftype} files like 'drop_%{cls.id}%'
|
|
1478
|
+
''')
|
|
1479
|
+
files = list(self.cur)
|
|
1480
|
+
assert len(files) == 0
|
|
1481
|
+
|
|
1482
|
+
# Upload files
|
|
1483
|
+
self.cur.execute(
|
|
1484
|
+
f'upload {ftype} file to "drop_test_1_{cls.id}.ipynb" from "{nb}"',
|
|
1485
|
+
)
|
|
1486
|
+
self.cur.execute(
|
|
1487
|
+
f'upload {ftype} file to "drop_test_2_{cls.id}.ipynb" from "{nb}"',
|
|
1488
|
+
)
|
|
1489
|
+
|
|
1490
|
+
# Make sure files are there
|
|
1491
|
+
self.cur.execute(f'''
|
|
1492
|
+
show {ftype} files like 'drop_%{cls.id}%'
|
|
1493
|
+
''')
|
|
1494
|
+
files = list(self.cur)
|
|
1495
|
+
assert len(files) == 2
|
|
1496
|
+
assert list(sorted(x[0] for x in files)) == [
|
|
1497
|
+
f'drop_test_1_{cls.id}.ipynb',
|
|
1498
|
+
f'drop_test_2_{cls.id}.ipynb',
|
|
1499
|
+
]
|
|
1500
|
+
|
|
1501
|
+
# Drop 1 file
|
|
1502
|
+
self.cur.execute(f'''
|
|
1503
|
+
drop {ftype} file 'drop_test_1_{cls.id}.ipynb'
|
|
1504
|
+
''')
|
|
1505
|
+
|
|
1506
|
+
# Make sure 1 file is there
|
|
1507
|
+
self.cur.execute(f'''
|
|
1508
|
+
show {ftype} files like 'drop_%{cls.id}%'
|
|
1509
|
+
''')
|
|
1510
|
+
files = list(self.cur)
|
|
1511
|
+
assert len(files) == 1
|
|
1512
|
+
assert list(x[0] for x in files) == [f'drop_test_2_{cls.id}.ipynb']
|
|
1513
|
+
|
|
1514
|
+
# Drop 2nd file
|
|
1515
|
+
self.cur.execute(f'''
|
|
1516
|
+
drop {ftype} file 'drop_test_2_{cls.id}.ipynb'
|
|
1517
|
+
''')
|
|
1518
|
+
|
|
1519
|
+
# Make sure no files are there
|
|
1520
|
+
self.cur.execute(f'''
|
|
1521
|
+
show {ftype} files like 'drop_%{cls.id}%'
|
|
1522
|
+
''')
|
|
1523
|
+
files = list(self.cur)
|
|
1524
|
+
assert len(files) == 0
|
|
@@ -414,8 +414,12 @@ class TestStage(unittest.TestCase):
|
|
|
414
414
|
with self.assertRaises(OSError):
|
|
415
415
|
st.upload_file(TEST_DIR / 'test.sql', 'upload_test.sql')
|
|
416
416
|
|
|
417
|
-
# Force overwrite with new content
|
|
418
|
-
f = st.upload_file(
|
|
417
|
+
# Force overwrite with new content; use file object this time
|
|
418
|
+
f = st.upload_file(
|
|
419
|
+
open(TEST_DIR / 'test2.sql', 'r'),
|
|
420
|
+
'upload_test.sql',
|
|
421
|
+
overwrite=True,
|
|
422
|
+
)
|
|
419
423
|
assert str(f.path) == 'upload_test.sql'
|
|
420
424
|
assert f.type == 'file'
|
|
421
425
|
|
|
@@ -1093,6 +1097,50 @@ class TestFileSpaces(unittest.TestCase):
|
|
|
1093
1097
|
# Cleanup
|
|
1094
1098
|
space.remove('upload_test.ipynb')
|
|
1095
1099
|
|
|
1100
|
+
def test_upload_file_io(self):
|
|
1101
|
+
for space in [self.personal_space, self.shared_space]:
|
|
1102
|
+
root = space.info('/')
|
|
1103
|
+
assert str(root.path) == '/'
|
|
1104
|
+
assert root.type == 'directory'
|
|
1105
|
+
|
|
1106
|
+
# Upload files
|
|
1107
|
+
f = space.upload_file(
|
|
1108
|
+
open(TEST_DIR / 'test.ipynb', 'r'),
|
|
1109
|
+
'upload_test.ipynb',
|
|
1110
|
+
)
|
|
1111
|
+
assert str(f.path) == 'upload_test.ipynb'
|
|
1112
|
+
assert f.type == 'notebook'
|
|
1113
|
+
|
|
1114
|
+
# Download and compare to original
|
|
1115
|
+
txt = f.download(encoding='utf-8')
|
|
1116
|
+
assert txt == open(TEST_DIR / 'test.ipynb').read()
|
|
1117
|
+
|
|
1118
|
+
# Make sure we can't overwrite
|
|
1119
|
+
with self.assertRaises(OSError):
|
|
1120
|
+
space.upload_file(
|
|
1121
|
+
open(TEST_DIR / 'test.ipynb', 'r'),
|
|
1122
|
+
'upload_test.ipynb',
|
|
1123
|
+
)
|
|
1124
|
+
|
|
1125
|
+
# Force overwrite with new content
|
|
1126
|
+
f = space.upload_file(
|
|
1127
|
+
open(TEST_DIR / 'test2.ipynb', 'r'),
|
|
1128
|
+
'upload_test.ipynb', overwrite=True,
|
|
1129
|
+
)
|
|
1130
|
+
assert str(f.path) == 'upload_test.ipynb'
|
|
1131
|
+
assert f.type == 'notebook'
|
|
1132
|
+
|
|
1133
|
+
# Verify new content
|
|
1134
|
+
txt = f.download(encoding='utf-8')
|
|
1135
|
+
assert txt == open(TEST_DIR / 'test2.ipynb').read()
|
|
1136
|
+
|
|
1137
|
+
# Make sure we can't upload a folder
|
|
1138
|
+
with self.assertRaises(s2.ManagementError):
|
|
1139
|
+
space.upload_folder(TEST_DIR, 'test')
|
|
1140
|
+
|
|
1141
|
+
# Cleanup
|
|
1142
|
+
space.remove('upload_test.ipynb')
|
|
1143
|
+
|
|
1096
1144
|
def test_open(self):
|
|
1097
1145
|
for space in [self.personal_space, self.shared_space]:
|
|
1098
1146
|
# See if error is raised for non-existent file
|
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
_singlestoredb_accel.pyd,sha256=
|
|
2
|
-
singlestoredb/__init__.py,sha256=
|
|
1
|
+
_singlestoredb_accel.pyd,sha256=eIcqHNvo41uHe02n94fJEzSJyXdegzvb1VHa86AULsM,59392
|
|
2
|
+
singlestoredb/__init__.py,sha256=dlGP3oKt-x7HySro64o0v_fGXAnosKaQsSoDYQc42mo,1712
|
|
3
3
|
singlestoredb/auth.py,sha256=RmYiH0Wlc2RXc4pTlRMysxtBI445ggCIwojWKC_eDLE,7844
|
|
4
4
|
singlestoredb/config.py,sha256=5OY9RDlGBrBWAHsGrg0X4v0y_QwNb7G6FSQooMgT8rY,13032
|
|
5
5
|
singlestoredb/connection.py,sha256=Ty_idVYH50Qx-j8WXy7NeB-DYLAcpdjGYTrTHkKzG9U,47309
|
|
@@ -41,7 +41,7 @@ singlestoredb/fusion/handlers/files.py,sha256=pCx1sqnjPtQrp39rv_V4RX9CVtj6uSiL6H
|
|
|
41
41
|
singlestoredb/fusion/handlers/job.py,sha256=3enfxHwERH7T4u0FEwOPN0IL0GtepaCYgEsisiy3Df4,21753
|
|
42
42
|
singlestoredb/fusion/handlers/models.py,sha256=XWaPJQc3GQIOAcjNcxBSGUBJ3xu2qkzQ4ILa40TFQmY,6486
|
|
43
43
|
singlestoredb/fusion/handlers/stage.py,sha256=PP-SSP204lwpmnycSXXSmFPzoN535JVuwglDCbaQ8Lw,14789
|
|
44
|
-
singlestoredb/fusion/handlers/utils.py,sha256=
|
|
44
|
+
singlestoredb/fusion/handlers/utils.py,sha256=nV2lSzKhv7CzM7I_uIh5kmDV0Ec6VeeKoHczx5pVNcw,11009
|
|
45
45
|
singlestoredb/fusion/handlers/workspace.py,sha256=NxoEY5xd5lCQmXiim4nhAYCL0agHo1H_rGPpqa31hiw,28397
|
|
46
46
|
singlestoredb/http/__init__.py,sha256=4cEDvLloGc3LSpU-PnIwacyu0n5oIIIE6xk2SPyWD_w,939
|
|
47
47
|
singlestoredb/http/connection.py,sha256=dqXX5SSn6EOV7q0Ehh0E525eLT__B_jJLG-7jgVjbXg,40861
|
|
@@ -52,13 +52,13 @@ singlestoredb/management/__init__.py,sha256=A66ZnFyX--PsAZ2tvtYUfIUBvVGDBFQsnVc6
|
|
|
52
52
|
singlestoredb/management/billing_usage.py,sha256=0UHFSPCrN0nyeGFFM-HXS3NP8pYmYo2BCCahDEPXvzg,3883
|
|
53
53
|
singlestoredb/management/cluster.py,sha256=auBzNYIXvnI6rq3DNpPgJhwWoT6JsyZRikjpON23Pxg,14867
|
|
54
54
|
singlestoredb/management/export.py,sha256=2dCvnTnkxVI-arX3_375DtWzHkI1YNK5zaFYdHKE4cs,5277
|
|
55
|
-
singlestoredb/management/files.py,sha256=
|
|
55
|
+
singlestoredb/management/files.py,sha256=Z9GpS2EHf9atE8kJdz1vJtsiT80O6TV00MPhqyXfAAw,31579
|
|
56
56
|
singlestoredb/management/job.py,sha256=Npfe1JLYJlggGBrXLniPKwKUKF1i3alvSY1SFtvauSs,25498
|
|
57
57
|
singlestoredb/management/manager.py,sha256=8zU0d7NG83PYMhoAs2JriTqbqh-R2tLX7VZoeZtcogY,9148
|
|
58
58
|
singlestoredb/management/organization.py,sha256=JBsNC4R3boUKdYvyCZyfGoVMC1mD6SPuMI1UssBVoOM,5611
|
|
59
59
|
singlestoredb/management/region.py,sha256=oGoLLS88dE1GmY7GCc0BV7X3f7bWwKQyeXOVBFmK9Pk,1678
|
|
60
60
|
singlestoredb/management/utils.py,sha256=BP-Wb8Sg16GbdLI_DeBz-3ttMklz6ZjYyMOz-sxElz8,13594
|
|
61
|
-
singlestoredb/management/workspace.py,sha256=
|
|
61
|
+
singlestoredb/management/workspace.py,sha256=D9DzpeWU7xFjpj8bBYiXyasjVYVANeYjTzgkz2aKvJQ,57984
|
|
62
62
|
singlestoredb/mysql/__init__.py,sha256=CbpwzNUJPAmKPpIobC0-ugBta_RgHCMq7X7N75QLReY,4669
|
|
63
63
|
singlestoredb/mysql/_auth.py,sha256=YaqqyvAHmeraBv3BM207rNveUVPM-mPnW20ts_ynVWg,8341
|
|
64
64
|
singlestoredb/mysql/charset.py,sha256=mnCdMpvdub1S2mm2PSk2j5JddgsWRjsVLtGx-y9TskE,10724
|
|
@@ -120,9 +120,9 @@ singlestoredb/tests/test_dbapi.py,sha256=cNJoTEZvYG7ckcwT7xqlkJX-2TDEYGTDDU1Iguc
|
|
|
120
120
|
singlestoredb/tests/test_exceptions.py,sha256=vscMYmdOJr0JmkTAJrNI2w0Q96Nfugjkrt5_lYnw8i0,1176
|
|
121
121
|
singlestoredb/tests/test_ext_func.py,sha256=gQErR-wAN8BqLNG5U4pNbg4qkQEo6Re8Hd9_Ztqo1RM,38550
|
|
122
122
|
singlestoredb/tests/test_ext_func_data.py,sha256=9kn8BWmCjkbnP6hSbFhmhcdW4OmVT-GSvBTIzFBLEys,48796
|
|
123
|
-
singlestoredb/tests/test_fusion.py,sha256=
|
|
123
|
+
singlestoredb/tests/test_fusion.py,sha256=S0Jk2NrcOitqM98r5fosHGbZ1sCZ2uxar5t48v-uOD0,52045
|
|
124
124
|
singlestoredb/tests/test_http.py,sha256=7hwXe61hlUes3nji0MTTZweo94tJAlJ-vA5ct9geXFQ,8868
|
|
125
|
-
singlestoredb/tests/test_management.py,sha256=
|
|
125
|
+
singlestoredb/tests/test_management.py,sha256=EJY-JDFNS83_pM0KKJlEHoRiEZWHxjK2XWtM0YAy63U,46462
|
|
126
126
|
singlestoredb/tests/test_plugin.py,sha256=P1nXLnTafaHkHN-6bVbGryxTu7OWJPU9SYFZ_WQUwq8,845
|
|
127
127
|
singlestoredb/tests/test_results.py,sha256=Zg1ynZFRZqalAMfNLOU5C6BDXaox6JxrKm_XZwVNFcg,6753
|
|
128
128
|
singlestoredb/tests/test_types.py,sha256=YeVE6KPqlqzJke-4hbRmc8ko1E7RLHu5S8qLg04Bl5Y,4632
|
|
@@ -141,9 +141,9 @@ singlestoredb/utils/results.py,sha256=wR70LhCqlobniZf52r67zYLBOKjWHQm68NAskdRQND
|
|
|
141
141
|
singlestoredb/utils/xdict.py,sha256=-wi1lSPTnY99fhVMBhPKJ8cCsQhNG4GMUfkEBDKYgCw,13321
|
|
142
142
|
sqlx/__init__.py,sha256=4Sdn8HN-Hf8v0_wCt60DCckCg8BvgM3-9r4YVfZycRE,89
|
|
143
143
|
sqlx/magic.py,sha256=6VBlotgjautjev599tHaTYOfcfOA9m6gV_-P1_Qc4lI,3622
|
|
144
|
-
singlestoredb-1.12.
|
|
145
|
-
singlestoredb-1.12.
|
|
146
|
-
singlestoredb-1.12.
|
|
147
|
-
singlestoredb-1.12.
|
|
148
|
-
singlestoredb-1.12.
|
|
149
|
-
singlestoredb-1.12.
|
|
144
|
+
singlestoredb-1.12.2.dist-info/LICENSE,sha256=Bojenzui8aPNjlF3w4ojguDP7sTf8vFV_9Gc2UAG1sg,11542
|
|
145
|
+
singlestoredb-1.12.2.dist-info/METADATA,sha256=cWTppHIaZQ4oy5fvfS_FWlVtfVPcyA2Bk9F1kb9kfgM,5778
|
|
146
|
+
singlestoredb-1.12.2.dist-info/WHEEL,sha256=UyMHzmWA0xVqVPKfTiLs2eN3OWWZUl-kQemNbpIqlKo,100
|
|
147
|
+
singlestoredb-1.12.2.dist-info/entry_points.txt,sha256=bSLaTWB5zGjpVYPAaI46MkkDup0su-eb3uAhCNYuRV0,48
|
|
148
|
+
singlestoredb-1.12.2.dist-info/top_level.txt,sha256=lA65Vf4qAMfg_s1oG3LEO90h4t1Z-SPDbRqkevI3bSY,40
|
|
149
|
+
singlestoredb-1.12.2.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|