singlestoredb 1.16.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- singlestoredb/__init__.py +75 -0
- singlestoredb/ai/__init__.py +2 -0
- singlestoredb/ai/chat.py +139 -0
- singlestoredb/ai/embeddings.py +128 -0
- singlestoredb/alchemy/__init__.py +90 -0
- singlestoredb/apps/__init__.py +3 -0
- singlestoredb/apps/_cloud_functions.py +90 -0
- singlestoredb/apps/_config.py +72 -0
- singlestoredb/apps/_connection_info.py +18 -0
- singlestoredb/apps/_dashboards.py +47 -0
- singlestoredb/apps/_process.py +32 -0
- singlestoredb/apps/_python_udfs.py +100 -0
- singlestoredb/apps/_stdout_supress.py +30 -0
- singlestoredb/apps/_uvicorn_util.py +36 -0
- singlestoredb/auth.py +245 -0
- singlestoredb/config.py +484 -0
- singlestoredb/connection.py +1487 -0
- singlestoredb/converters.py +950 -0
- singlestoredb/docstring/__init__.py +33 -0
- singlestoredb/docstring/attrdoc.py +126 -0
- singlestoredb/docstring/common.py +230 -0
- singlestoredb/docstring/epydoc.py +267 -0
- singlestoredb/docstring/google.py +412 -0
- singlestoredb/docstring/numpydoc.py +562 -0
- singlestoredb/docstring/parser.py +100 -0
- singlestoredb/docstring/py.typed +1 -0
- singlestoredb/docstring/rest.py +256 -0
- singlestoredb/docstring/tests/__init__.py +1 -0
- singlestoredb/docstring/tests/_pydoctor.py +21 -0
- singlestoredb/docstring/tests/test_epydoc.py +729 -0
- singlestoredb/docstring/tests/test_google.py +1007 -0
- singlestoredb/docstring/tests/test_numpydoc.py +1100 -0
- singlestoredb/docstring/tests/test_parse_from_object.py +109 -0
- singlestoredb/docstring/tests/test_parser.py +248 -0
- singlestoredb/docstring/tests/test_rest.py +547 -0
- singlestoredb/docstring/tests/test_util.py +70 -0
- singlestoredb/docstring/util.py +141 -0
- singlestoredb/exceptions.py +120 -0
- singlestoredb/functions/__init__.py +16 -0
- singlestoredb/functions/decorator.py +201 -0
- singlestoredb/functions/dtypes.py +1793 -0
- singlestoredb/functions/ext/__init__.py +1 -0
- singlestoredb/functions/ext/arrow.py +375 -0
- singlestoredb/functions/ext/asgi.py +2133 -0
- singlestoredb/functions/ext/json.py +420 -0
- singlestoredb/functions/ext/mmap.py +413 -0
- singlestoredb/functions/ext/rowdat_1.py +724 -0
- singlestoredb/functions/ext/timer.py +89 -0
- singlestoredb/functions/ext/utils.py +218 -0
- singlestoredb/functions/signature.py +1578 -0
- singlestoredb/functions/typing/__init__.py +41 -0
- singlestoredb/functions/typing/numpy.py +20 -0
- singlestoredb/functions/typing/pandas.py +2 -0
- singlestoredb/functions/typing/polars.py +2 -0
- singlestoredb/functions/typing/pyarrow.py +2 -0
- singlestoredb/functions/utils.py +421 -0
- singlestoredb/fusion/__init__.py +11 -0
- singlestoredb/fusion/graphql.py +213 -0
- singlestoredb/fusion/handler.py +916 -0
- singlestoredb/fusion/handlers/__init__.py +0 -0
- singlestoredb/fusion/handlers/export.py +525 -0
- singlestoredb/fusion/handlers/files.py +690 -0
- singlestoredb/fusion/handlers/job.py +660 -0
- singlestoredb/fusion/handlers/models.py +250 -0
- singlestoredb/fusion/handlers/stage.py +502 -0
- singlestoredb/fusion/handlers/utils.py +324 -0
- singlestoredb/fusion/handlers/workspace.py +956 -0
- singlestoredb/fusion/registry.py +249 -0
- singlestoredb/fusion/result.py +399 -0
- singlestoredb/http/__init__.py +27 -0
- singlestoredb/http/connection.py +1267 -0
- singlestoredb/magics/__init__.py +34 -0
- singlestoredb/magics/run_personal.py +137 -0
- singlestoredb/magics/run_shared.py +134 -0
- singlestoredb/management/__init__.py +9 -0
- singlestoredb/management/billing_usage.py +148 -0
- singlestoredb/management/cluster.py +462 -0
- singlestoredb/management/export.py +295 -0
- singlestoredb/management/files.py +1102 -0
- singlestoredb/management/inference_api.py +105 -0
- singlestoredb/management/job.py +887 -0
- singlestoredb/management/manager.py +373 -0
- singlestoredb/management/organization.py +226 -0
- singlestoredb/management/region.py +169 -0
- singlestoredb/management/utils.py +423 -0
- singlestoredb/management/workspace.py +1927 -0
- singlestoredb/mysql/__init__.py +177 -0
- singlestoredb/mysql/_auth.py +298 -0
- singlestoredb/mysql/charset.py +214 -0
- singlestoredb/mysql/connection.py +2032 -0
- singlestoredb/mysql/constants/CLIENT.py +38 -0
- singlestoredb/mysql/constants/COMMAND.py +32 -0
- singlestoredb/mysql/constants/CR.py +78 -0
- singlestoredb/mysql/constants/ER.py +474 -0
- singlestoredb/mysql/constants/EXTENDED_TYPE.py +3 -0
- singlestoredb/mysql/constants/FIELD_TYPE.py +48 -0
- singlestoredb/mysql/constants/FLAG.py +15 -0
- singlestoredb/mysql/constants/SERVER_STATUS.py +10 -0
- singlestoredb/mysql/constants/VECTOR_TYPE.py +6 -0
- singlestoredb/mysql/constants/__init__.py +0 -0
- singlestoredb/mysql/converters.py +271 -0
- singlestoredb/mysql/cursors.py +896 -0
- singlestoredb/mysql/err.py +92 -0
- singlestoredb/mysql/optionfile.py +20 -0
- singlestoredb/mysql/protocol.py +450 -0
- singlestoredb/mysql/tests/__init__.py +19 -0
- singlestoredb/mysql/tests/base.py +126 -0
- singlestoredb/mysql/tests/conftest.py +37 -0
- singlestoredb/mysql/tests/test_DictCursor.py +132 -0
- singlestoredb/mysql/tests/test_SSCursor.py +141 -0
- singlestoredb/mysql/tests/test_basic.py +452 -0
- singlestoredb/mysql/tests/test_connection.py +851 -0
- singlestoredb/mysql/tests/test_converters.py +58 -0
- singlestoredb/mysql/tests/test_cursor.py +141 -0
- singlestoredb/mysql/tests/test_err.py +16 -0
- singlestoredb/mysql/tests/test_issues.py +514 -0
- singlestoredb/mysql/tests/test_load_local.py +75 -0
- singlestoredb/mysql/tests/test_nextset.py +88 -0
- singlestoredb/mysql/tests/test_optionfile.py +27 -0
- singlestoredb/mysql/tests/thirdparty/__init__.py +6 -0
- singlestoredb/mysql/tests/thirdparty/test_MySQLdb/__init__.py +9 -0
- singlestoredb/mysql/tests/thirdparty/test_MySQLdb/capabilities.py +323 -0
- singlestoredb/mysql/tests/thirdparty/test_MySQLdb/dbapi20.py +865 -0
- singlestoredb/mysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_capabilities.py +110 -0
- singlestoredb/mysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_dbapi20.py +224 -0
- singlestoredb/mysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_nonstandard.py +101 -0
- singlestoredb/mysql/times.py +23 -0
- singlestoredb/notebook/__init__.py +16 -0
- singlestoredb/notebook/_objects.py +213 -0
- singlestoredb/notebook/_portal.py +352 -0
- singlestoredb/py.typed +0 -0
- singlestoredb/pytest.py +352 -0
- singlestoredb/server/__init__.py +0 -0
- singlestoredb/server/docker.py +452 -0
- singlestoredb/server/free_tier.py +267 -0
- singlestoredb/tests/__init__.py +0 -0
- singlestoredb/tests/alltypes.sql +307 -0
- singlestoredb/tests/alltypes_no_nulls.sql +208 -0
- singlestoredb/tests/empty.sql +0 -0
- singlestoredb/tests/ext_funcs/__init__.py +702 -0
- singlestoredb/tests/local_infile.csv +3 -0
- singlestoredb/tests/test.ipynb +18 -0
- singlestoredb/tests/test.sql +680 -0
- singlestoredb/tests/test2.ipynb +18 -0
- singlestoredb/tests/test2.sql +1 -0
- singlestoredb/tests/test_basics.py +1332 -0
- singlestoredb/tests/test_config.py +318 -0
- singlestoredb/tests/test_connection.py +3103 -0
- singlestoredb/tests/test_dbapi.py +27 -0
- singlestoredb/tests/test_exceptions.py +45 -0
- singlestoredb/tests/test_ext_func.py +1472 -0
- singlestoredb/tests/test_ext_func_data.py +1101 -0
- singlestoredb/tests/test_fusion.py +1527 -0
- singlestoredb/tests/test_http.py +288 -0
- singlestoredb/tests/test_management.py +1599 -0
- singlestoredb/tests/test_plugin.py +33 -0
- singlestoredb/tests/test_results.py +171 -0
- singlestoredb/tests/test_types.py +132 -0
- singlestoredb/tests/test_udf.py +737 -0
- singlestoredb/tests/test_udf_returns.py +459 -0
- singlestoredb/tests/test_vectorstore.py +51 -0
- singlestoredb/tests/test_xdict.py +333 -0
- singlestoredb/tests/utils.py +141 -0
- singlestoredb/types.py +373 -0
- singlestoredb/utils/__init__.py +0 -0
- singlestoredb/utils/config.py +950 -0
- singlestoredb/utils/convert_rows.py +69 -0
- singlestoredb/utils/debug.py +13 -0
- singlestoredb/utils/dtypes.py +205 -0
- singlestoredb/utils/events.py +65 -0
- singlestoredb/utils/mogrify.py +151 -0
- singlestoredb/utils/results.py +585 -0
- singlestoredb/utils/xdict.py +425 -0
- singlestoredb/vectorstore.py +192 -0
- singlestoredb/warnings.py +5 -0
- singlestoredb-1.16.1.dist-info/METADATA +165 -0
- singlestoredb-1.16.1.dist-info/RECORD +183 -0
- singlestoredb-1.16.1.dist-info/WHEEL +5 -0
- singlestoredb-1.16.1.dist-info/entry_points.txt +2 -0
- singlestoredb-1.16.1.dist-info/licenses/LICENSE +201 -0
- singlestoredb-1.16.1.dist-info/top_level.txt +3 -0
- sqlx/__init__.py +4 -0
- sqlx/magic.py +113 -0
|
@@ -0,0 +1,1102 @@
|
|
|
1
|
+
#!/usr/bin/env python
|
|
2
|
+
"""SingleStore Cloud Files Management."""
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import datetime
|
|
6
|
+
import glob
|
|
7
|
+
import io
|
|
8
|
+
import os
|
|
9
|
+
import re
|
|
10
|
+
from abc import ABC
|
|
11
|
+
from abc import abstractmethod
|
|
12
|
+
from typing import Any
|
|
13
|
+
from typing import Dict
|
|
14
|
+
from typing import List
|
|
15
|
+
from typing import Optional
|
|
16
|
+
from typing import Union
|
|
17
|
+
|
|
18
|
+
from .. import config
|
|
19
|
+
from ..exceptions import ManagementError
|
|
20
|
+
from .manager import Manager
|
|
21
|
+
from .utils import PathLike
|
|
22
|
+
from .utils import to_datetime
|
|
23
|
+
from .utils import vars_to_str
|
|
24
|
+
|
|
25
|
+
PERSONAL_SPACE = 'personal'
|
|
26
|
+
SHARED_SPACE = 'shared'
|
|
27
|
+
MODELS_SPACE = 'models'
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class FilesObject(object):
|
|
31
|
+
"""
|
|
32
|
+
File / folder object.
|
|
33
|
+
|
|
34
|
+
It can belong to either a workspace stage or personal/shared space.
|
|
35
|
+
|
|
36
|
+
This object is not instantiated directly. It is used in the results
|
|
37
|
+
of various operations in ``WorkspaceGroup.stage``, ``FilesManager.personal_space``,
|
|
38
|
+
``FilesManager.shared_space`` and ``FilesManager.models_space`` methods.
|
|
39
|
+
|
|
40
|
+
"""
|
|
41
|
+
|
|
42
|
+
def __init__(
|
|
43
|
+
self,
|
|
44
|
+
name: str,
|
|
45
|
+
path: str,
|
|
46
|
+
size: int,
|
|
47
|
+
type: str,
|
|
48
|
+
format: str,
|
|
49
|
+
mimetype: str,
|
|
50
|
+
created: Optional[datetime.datetime],
|
|
51
|
+
last_modified: Optional[datetime.datetime],
|
|
52
|
+
writable: bool,
|
|
53
|
+
content: Optional[List[str]] = None,
|
|
54
|
+
):
|
|
55
|
+
#: Name of file / folder
|
|
56
|
+
self.name = name
|
|
57
|
+
|
|
58
|
+
if type == 'directory':
|
|
59
|
+
path = re.sub(r'/*$', r'', str(path)) + '/'
|
|
60
|
+
|
|
61
|
+
#: Path of file / folder
|
|
62
|
+
self.path = path
|
|
63
|
+
|
|
64
|
+
#: Size of the object (in bytes)
|
|
65
|
+
self.size = size
|
|
66
|
+
|
|
67
|
+
#: Data type: file or directory
|
|
68
|
+
self.type = type
|
|
69
|
+
|
|
70
|
+
#: Data format
|
|
71
|
+
self.format = format
|
|
72
|
+
|
|
73
|
+
#: Mime type
|
|
74
|
+
self.mimetype = mimetype
|
|
75
|
+
|
|
76
|
+
#: Datetime the object was created
|
|
77
|
+
self.created_at = created
|
|
78
|
+
|
|
79
|
+
#: Datetime the object was modified last
|
|
80
|
+
self.last_modified_at = last_modified
|
|
81
|
+
|
|
82
|
+
#: Is the object writable?
|
|
83
|
+
self.writable = writable
|
|
84
|
+
|
|
85
|
+
#: Contents of a directory
|
|
86
|
+
self.content: List[str] = content or []
|
|
87
|
+
|
|
88
|
+
self._location: Optional[FileLocation] = None
|
|
89
|
+
|
|
90
|
+
@classmethod
|
|
91
|
+
def from_dict(
|
|
92
|
+
cls,
|
|
93
|
+
obj: Dict[str, Any],
|
|
94
|
+
location: FileLocation,
|
|
95
|
+
) -> FilesObject:
|
|
96
|
+
"""
|
|
97
|
+
Construct a FilesObject from a dictionary of values.
|
|
98
|
+
|
|
99
|
+
Parameters
|
|
100
|
+
----------
|
|
101
|
+
obj : dict
|
|
102
|
+
Dictionary of values
|
|
103
|
+
location : FileLocation
|
|
104
|
+
FileLocation object to use as the parent
|
|
105
|
+
|
|
106
|
+
Returns
|
|
107
|
+
-------
|
|
108
|
+
:class:`FilesObject`
|
|
109
|
+
|
|
110
|
+
"""
|
|
111
|
+
out = cls(
|
|
112
|
+
name=obj['name'],
|
|
113
|
+
path=obj['path'],
|
|
114
|
+
size=obj['size'],
|
|
115
|
+
type=obj['type'],
|
|
116
|
+
format=obj['format'],
|
|
117
|
+
mimetype=obj['mimetype'],
|
|
118
|
+
created=to_datetime(obj.get('created')),
|
|
119
|
+
last_modified=to_datetime(obj.get('last_modified')),
|
|
120
|
+
writable=bool(obj['writable']),
|
|
121
|
+
)
|
|
122
|
+
out._location = location
|
|
123
|
+
return out
|
|
124
|
+
|
|
125
|
+
def __str__(self) -> str:
|
|
126
|
+
"""Return string representation."""
|
|
127
|
+
return vars_to_str(self)
|
|
128
|
+
|
|
129
|
+
def __repr__(self) -> str:
|
|
130
|
+
"""Return string representation."""
|
|
131
|
+
return str(self)
|
|
132
|
+
|
|
133
|
+
def open(
|
|
134
|
+
self,
|
|
135
|
+
mode: str = 'r',
|
|
136
|
+
encoding: Optional[str] = None,
|
|
137
|
+
) -> Union[io.StringIO, io.BytesIO]:
|
|
138
|
+
"""
|
|
139
|
+
Open a file path for reading or writing.
|
|
140
|
+
|
|
141
|
+
Parameters
|
|
142
|
+
----------
|
|
143
|
+
mode : str, optional
|
|
144
|
+
The read / write mode. The following modes are supported:
|
|
145
|
+
* 'r' open for reading (default)
|
|
146
|
+
* 'w' open for writing, truncating the file first
|
|
147
|
+
* 'x' create a new file and open it for writing
|
|
148
|
+
The data type can be specified by adding one of the following:
|
|
149
|
+
* 'b' binary mode
|
|
150
|
+
* 't' text mode (default)
|
|
151
|
+
encoding : str, optional
|
|
152
|
+
The string encoding to use for text
|
|
153
|
+
|
|
154
|
+
Returns
|
|
155
|
+
-------
|
|
156
|
+
FilesObjectBytesReader - 'rb' or 'b' mode
|
|
157
|
+
FilesObjectBytesWriter - 'wb' or 'xb' mode
|
|
158
|
+
FilesObjectTextReader - 'r' or 'rt' mode
|
|
159
|
+
FilesObjectTextWriter - 'w', 'x', 'wt' or 'xt' mode
|
|
160
|
+
|
|
161
|
+
"""
|
|
162
|
+
if self._location is None:
|
|
163
|
+
raise ManagementError(
|
|
164
|
+
msg='No FileLocation object is associated with this object.',
|
|
165
|
+
)
|
|
166
|
+
|
|
167
|
+
if self.is_dir():
|
|
168
|
+
raise IsADirectoryError(
|
|
169
|
+
f'directories can not be read or written: {self.path}',
|
|
170
|
+
)
|
|
171
|
+
|
|
172
|
+
return self._location.open(self.path, mode=mode, encoding=encoding)
|
|
173
|
+
|
|
174
|
+
def download(
|
|
175
|
+
self,
|
|
176
|
+
local_path: Optional[PathLike] = None,
|
|
177
|
+
*,
|
|
178
|
+
overwrite: bool = False,
|
|
179
|
+
encoding: Optional[str] = None,
|
|
180
|
+
) -> Optional[Union[bytes, str]]:
|
|
181
|
+
"""
|
|
182
|
+
Download the content of a file path.
|
|
183
|
+
|
|
184
|
+
Parameters
|
|
185
|
+
----------
|
|
186
|
+
local_path : Path or str
|
|
187
|
+
Path to local file target location
|
|
188
|
+
overwrite : bool, optional
|
|
189
|
+
Should an existing file be overwritten if it exists?
|
|
190
|
+
encoding : str, optional
|
|
191
|
+
Encoding used to convert the resulting data
|
|
192
|
+
|
|
193
|
+
Returns
|
|
194
|
+
-------
|
|
195
|
+
bytes or str or None
|
|
196
|
+
|
|
197
|
+
"""
|
|
198
|
+
if self._location is None:
|
|
199
|
+
raise ManagementError(
|
|
200
|
+
msg='No FileLocation object is associated with this object.',
|
|
201
|
+
)
|
|
202
|
+
|
|
203
|
+
return self._location.download_file(
|
|
204
|
+
self.path, local_path=local_path,
|
|
205
|
+
overwrite=overwrite, encoding=encoding,
|
|
206
|
+
)
|
|
207
|
+
|
|
208
|
+
download_file = download
|
|
209
|
+
|
|
210
|
+
def remove(self) -> None:
|
|
211
|
+
"""Delete the file."""
|
|
212
|
+
if self._location is None:
|
|
213
|
+
raise ManagementError(
|
|
214
|
+
msg='No FileLocation object is associated with this object.',
|
|
215
|
+
)
|
|
216
|
+
|
|
217
|
+
if self.type == 'directory':
|
|
218
|
+
raise IsADirectoryError(
|
|
219
|
+
f'path is a directory; use rmdir or removedirs {self.path}',
|
|
220
|
+
)
|
|
221
|
+
|
|
222
|
+
self._location.remove(self.path)
|
|
223
|
+
|
|
224
|
+
def rmdir(self) -> None:
|
|
225
|
+
"""Delete the empty directory."""
|
|
226
|
+
if self._location is None:
|
|
227
|
+
raise ManagementError(
|
|
228
|
+
msg='No FileLocation object is associated with this object.',
|
|
229
|
+
)
|
|
230
|
+
|
|
231
|
+
if self.type != 'directory':
|
|
232
|
+
raise NotADirectoryError(
|
|
233
|
+
f'path is not a directory: {self.path}',
|
|
234
|
+
)
|
|
235
|
+
|
|
236
|
+
self._location.rmdir(self.path)
|
|
237
|
+
|
|
238
|
+
def removedirs(self) -> None:
|
|
239
|
+
"""Delete the directory recursively."""
|
|
240
|
+
if self._location is None:
|
|
241
|
+
raise ManagementError(
|
|
242
|
+
msg='No FileLocation object is associated with this object.',
|
|
243
|
+
)
|
|
244
|
+
|
|
245
|
+
if self.type != 'directory':
|
|
246
|
+
raise NotADirectoryError(
|
|
247
|
+
f'path is not a directory: {self.path}',
|
|
248
|
+
)
|
|
249
|
+
|
|
250
|
+
self._location.removedirs(self.path)
|
|
251
|
+
|
|
252
|
+
def rename(self, new_path: PathLike, *, overwrite: bool = False) -> None:
|
|
253
|
+
"""
|
|
254
|
+
Move the file to a new location.
|
|
255
|
+
|
|
256
|
+
Parameters
|
|
257
|
+
----------
|
|
258
|
+
new_path : Path or str
|
|
259
|
+
The new location of the file
|
|
260
|
+
overwrite : bool, optional
|
|
261
|
+
Should path be overwritten if it already exists?
|
|
262
|
+
|
|
263
|
+
"""
|
|
264
|
+
if self._location is None:
|
|
265
|
+
raise ManagementError(
|
|
266
|
+
msg='No FileLocation object is associated with this object.',
|
|
267
|
+
)
|
|
268
|
+
out = self._location.rename(self.path, new_path, overwrite=overwrite)
|
|
269
|
+
self.name = out.name
|
|
270
|
+
self.path = out.path
|
|
271
|
+
return None
|
|
272
|
+
|
|
273
|
+
def exists(self) -> bool:
|
|
274
|
+
"""Does the file / folder exist?"""
|
|
275
|
+
if self._location is None:
|
|
276
|
+
raise ManagementError(
|
|
277
|
+
msg='No FileLocation object is associated with this object.',
|
|
278
|
+
)
|
|
279
|
+
return self._location.exists(self.path)
|
|
280
|
+
|
|
281
|
+
def is_dir(self) -> bool:
|
|
282
|
+
"""Is the object a directory?"""
|
|
283
|
+
return self.type == 'directory'
|
|
284
|
+
|
|
285
|
+
def is_file(self) -> bool:
|
|
286
|
+
"""Is the object a file?"""
|
|
287
|
+
return self.type != 'directory'
|
|
288
|
+
|
|
289
|
+
def abspath(self) -> str:
|
|
290
|
+
"""Return the full path of the object."""
|
|
291
|
+
return str(self.path)
|
|
292
|
+
|
|
293
|
+
def basename(self) -> str:
|
|
294
|
+
"""Return the basename of the object."""
|
|
295
|
+
return self.name
|
|
296
|
+
|
|
297
|
+
def dirname(self) -> str:
|
|
298
|
+
"""Return the directory name of the object."""
|
|
299
|
+
return re.sub(r'/*$', r'', os.path.dirname(re.sub(r'/*$', r'', self.path))) + '/'
|
|
300
|
+
|
|
301
|
+
def getmtime(self) -> float:
|
|
302
|
+
"""Return the last modified datetime as a UNIX timestamp."""
|
|
303
|
+
if self.last_modified_at is None:
|
|
304
|
+
return 0.0
|
|
305
|
+
return self.last_modified_at.timestamp()
|
|
306
|
+
|
|
307
|
+
def getctime(self) -> float:
|
|
308
|
+
"""Return the creation datetime as a UNIX timestamp."""
|
|
309
|
+
if self.created_at is None:
|
|
310
|
+
return 0.0
|
|
311
|
+
return self.created_at.timestamp()
|
|
312
|
+
|
|
313
|
+
|
|
314
|
+
class FilesObjectTextWriter(io.StringIO):
|
|
315
|
+
"""StringIO wrapper for writing to FileLocation."""
|
|
316
|
+
|
|
317
|
+
def __init__(self, buffer: Optional[str], location: FileLocation, path: PathLike):
|
|
318
|
+
self._location = location
|
|
319
|
+
self._path = path
|
|
320
|
+
super().__init__(buffer)
|
|
321
|
+
|
|
322
|
+
def close(self) -> None:
|
|
323
|
+
"""Write the content to the path."""
|
|
324
|
+
self._location._upload(self.getvalue(), self._path)
|
|
325
|
+
super().close()
|
|
326
|
+
|
|
327
|
+
|
|
328
|
+
class FilesObjectTextReader(io.StringIO):
|
|
329
|
+
"""StringIO wrapper for reading from FileLocation."""
|
|
330
|
+
|
|
331
|
+
|
|
332
|
+
class FilesObjectBytesWriter(io.BytesIO):
|
|
333
|
+
"""BytesIO wrapper for writing to FileLocation."""
|
|
334
|
+
|
|
335
|
+
def __init__(self, buffer: bytes, location: FileLocation, path: PathLike):
|
|
336
|
+
self._location = location
|
|
337
|
+
self._path = path
|
|
338
|
+
super().__init__(buffer)
|
|
339
|
+
|
|
340
|
+
def close(self) -> None:
|
|
341
|
+
"""Write the content to the file path."""
|
|
342
|
+
self._location._upload(self.getvalue(), self._path)
|
|
343
|
+
super().close()
|
|
344
|
+
|
|
345
|
+
|
|
346
|
+
class FilesObjectBytesReader(io.BytesIO):
|
|
347
|
+
"""BytesIO wrapper for reading from FileLocation."""
|
|
348
|
+
|
|
349
|
+
|
|
350
|
+
class FileLocation(ABC):
|
|
351
|
+
|
|
352
|
+
@abstractmethod
|
|
353
|
+
def open(
|
|
354
|
+
self,
|
|
355
|
+
path: PathLike,
|
|
356
|
+
mode: str = 'r',
|
|
357
|
+
encoding: Optional[str] = None,
|
|
358
|
+
) -> Union[io.StringIO, io.BytesIO]:
|
|
359
|
+
pass
|
|
360
|
+
|
|
361
|
+
@abstractmethod
|
|
362
|
+
def upload_file(
|
|
363
|
+
self,
|
|
364
|
+
local_path: Union[PathLike, io.IOBase],
|
|
365
|
+
path: PathLike,
|
|
366
|
+
*,
|
|
367
|
+
overwrite: bool = False,
|
|
368
|
+
) -> FilesObject:
|
|
369
|
+
pass
|
|
370
|
+
|
|
371
|
+
@abstractmethod
|
|
372
|
+
def upload_folder(
|
|
373
|
+
self,
|
|
374
|
+
local_path: PathLike,
|
|
375
|
+
path: PathLike,
|
|
376
|
+
*,
|
|
377
|
+
overwrite: bool = False,
|
|
378
|
+
recursive: bool = True,
|
|
379
|
+
include_root: bool = False,
|
|
380
|
+
ignore: Optional[Union[PathLike, List[PathLike]]] = None,
|
|
381
|
+
) -> FilesObject:
|
|
382
|
+
pass
|
|
383
|
+
|
|
384
|
+
@abstractmethod
|
|
385
|
+
def _upload(
|
|
386
|
+
self,
|
|
387
|
+
content: Union[str, bytes, io.IOBase],
|
|
388
|
+
path: PathLike,
|
|
389
|
+
*,
|
|
390
|
+
overwrite: bool = False,
|
|
391
|
+
) -> FilesObject:
|
|
392
|
+
pass
|
|
393
|
+
|
|
394
|
+
@abstractmethod
|
|
395
|
+
def mkdir(self, path: PathLike, overwrite: bool = False) -> FilesObject:
|
|
396
|
+
pass
|
|
397
|
+
|
|
398
|
+
@abstractmethod
|
|
399
|
+
def rename(
|
|
400
|
+
self,
|
|
401
|
+
old_path: PathLike,
|
|
402
|
+
new_path: PathLike,
|
|
403
|
+
*,
|
|
404
|
+
overwrite: bool = False,
|
|
405
|
+
) -> FilesObject:
|
|
406
|
+
pass
|
|
407
|
+
|
|
408
|
+
@abstractmethod
|
|
409
|
+
def info(self, path: PathLike) -> FilesObject:
|
|
410
|
+
pass
|
|
411
|
+
|
|
412
|
+
@abstractmethod
|
|
413
|
+
def exists(self, path: PathLike) -> bool:
|
|
414
|
+
pass
|
|
415
|
+
|
|
416
|
+
@abstractmethod
|
|
417
|
+
def is_dir(self, path: PathLike) -> bool:
|
|
418
|
+
pass
|
|
419
|
+
|
|
420
|
+
@abstractmethod
|
|
421
|
+
def is_file(self, path: PathLike) -> bool:
|
|
422
|
+
pass
|
|
423
|
+
|
|
424
|
+
@abstractmethod
|
|
425
|
+
def listdir(
|
|
426
|
+
self,
|
|
427
|
+
path: PathLike = '/',
|
|
428
|
+
*,
|
|
429
|
+
recursive: bool = False,
|
|
430
|
+
) -> List[str]:
|
|
431
|
+
pass
|
|
432
|
+
|
|
433
|
+
@abstractmethod
|
|
434
|
+
def download_file(
|
|
435
|
+
self,
|
|
436
|
+
path: PathLike,
|
|
437
|
+
local_path: Optional[PathLike] = None,
|
|
438
|
+
*,
|
|
439
|
+
overwrite: bool = False,
|
|
440
|
+
encoding: Optional[str] = None,
|
|
441
|
+
) -> Optional[Union[bytes, str]]:
|
|
442
|
+
pass
|
|
443
|
+
|
|
444
|
+
@abstractmethod
|
|
445
|
+
def download_folder(
|
|
446
|
+
self,
|
|
447
|
+
path: PathLike,
|
|
448
|
+
local_path: PathLike = '.',
|
|
449
|
+
*,
|
|
450
|
+
overwrite: bool = False,
|
|
451
|
+
) -> None:
|
|
452
|
+
pass
|
|
453
|
+
|
|
454
|
+
@abstractmethod
|
|
455
|
+
def remove(self, path: PathLike) -> None:
|
|
456
|
+
pass
|
|
457
|
+
|
|
458
|
+
@abstractmethod
|
|
459
|
+
def removedirs(self, path: PathLike) -> None:
|
|
460
|
+
pass
|
|
461
|
+
|
|
462
|
+
@abstractmethod
|
|
463
|
+
def rmdir(self, path: PathLike) -> None:
|
|
464
|
+
pass
|
|
465
|
+
|
|
466
|
+
@abstractmethod
|
|
467
|
+
def __str__(self) -> str:
|
|
468
|
+
pass
|
|
469
|
+
|
|
470
|
+
@abstractmethod
|
|
471
|
+
def __repr__(self) -> str:
|
|
472
|
+
pass
|
|
473
|
+
|
|
474
|
+
|
|
475
|
+
class FilesManager(Manager):
|
|
476
|
+
"""
|
|
477
|
+
SingleStoreDB files manager.
|
|
478
|
+
|
|
479
|
+
This class should be instantiated using :func:`singlestoredb.manage_files`.
|
|
480
|
+
|
|
481
|
+
Parameters
|
|
482
|
+
----------
|
|
483
|
+
access_token : str, optional
|
|
484
|
+
The API key or other access token for the files management API
|
|
485
|
+
version : str, optional
|
|
486
|
+
Version of the API to use
|
|
487
|
+
base_url : str, optional
|
|
488
|
+
Base URL of the files management API
|
|
489
|
+
|
|
490
|
+
See Also
|
|
491
|
+
--------
|
|
492
|
+
:func:`singlestoredb.manage_files`
|
|
493
|
+
|
|
494
|
+
"""
|
|
495
|
+
|
|
496
|
+
#: Management API version if none is specified.
|
|
497
|
+
default_version = config.get_option('management.version') or 'v1'
|
|
498
|
+
|
|
499
|
+
#: Base URL if none is specified.
|
|
500
|
+
default_base_url = config.get_option('management.base_url') \
|
|
501
|
+
or 'https://api.singlestore.com'
|
|
502
|
+
|
|
503
|
+
#: Object type
|
|
504
|
+
obj_type = 'file'
|
|
505
|
+
|
|
506
|
+
@property
|
|
507
|
+
def personal_space(self) -> FileSpace:
|
|
508
|
+
"""Return the personal file space."""
|
|
509
|
+
return FileSpace(PERSONAL_SPACE, self)
|
|
510
|
+
|
|
511
|
+
@property
|
|
512
|
+
def shared_space(self) -> FileSpace:
|
|
513
|
+
"""Return the shared file space."""
|
|
514
|
+
return FileSpace(SHARED_SPACE, self)
|
|
515
|
+
|
|
516
|
+
@property
|
|
517
|
+
def models_space(self) -> FileSpace:
|
|
518
|
+
"""Return the models file space."""
|
|
519
|
+
return FileSpace(MODELS_SPACE, self)
|
|
520
|
+
|
|
521
|
+
|
|
522
|
+
def manage_files(
|
|
523
|
+
access_token: Optional[str] = None,
|
|
524
|
+
version: Optional[str] = None,
|
|
525
|
+
base_url: Optional[str] = None,
|
|
526
|
+
*,
|
|
527
|
+
organization_id: Optional[str] = None,
|
|
528
|
+
) -> FilesManager:
|
|
529
|
+
"""
|
|
530
|
+
Retrieve a SingleStoreDB files manager.
|
|
531
|
+
|
|
532
|
+
Parameters
|
|
533
|
+
----------
|
|
534
|
+
access_token : str, optional
|
|
535
|
+
The API key or other access token for the files management API
|
|
536
|
+
version : str, optional
|
|
537
|
+
Version of the API to use
|
|
538
|
+
base_url : str, optional
|
|
539
|
+
Base URL of the files management API
|
|
540
|
+
organization_id : str, optional
|
|
541
|
+
ID of organization, if using a JWT for authentication
|
|
542
|
+
|
|
543
|
+
Returns
|
|
544
|
+
-------
|
|
545
|
+
:class:`FilesManager`
|
|
546
|
+
|
|
547
|
+
"""
|
|
548
|
+
return FilesManager(
|
|
549
|
+
access_token=access_token, base_url=base_url,
|
|
550
|
+
version=version, organization_id=organization_id,
|
|
551
|
+
)
|
|
552
|
+
|
|
553
|
+
|
|
554
|
+
class FileSpace(FileLocation):
|
|
555
|
+
"""
|
|
556
|
+
FileSpace manager.
|
|
557
|
+
|
|
558
|
+
This object is not instantiated directly.
|
|
559
|
+
It is returned by ``FilesManager.personal_space``, ``FilesManager.shared_space``
|
|
560
|
+
or ``FileManger.models_space``.
|
|
561
|
+
|
|
562
|
+
"""
|
|
563
|
+
|
|
564
|
+
def __init__(self, location: str, manager: FilesManager):
|
|
565
|
+
self._location = location
|
|
566
|
+
self._manager = manager
|
|
567
|
+
|
|
568
|
+
def open(
|
|
569
|
+
self,
|
|
570
|
+
path: PathLike,
|
|
571
|
+
mode: str = 'r',
|
|
572
|
+
encoding: Optional[str] = None,
|
|
573
|
+
) -> Union[io.StringIO, io.BytesIO]:
|
|
574
|
+
"""
|
|
575
|
+
Open a file path for reading or writing.
|
|
576
|
+
|
|
577
|
+
Parameters
|
|
578
|
+
----------
|
|
579
|
+
path : Path or str
|
|
580
|
+
The file path to read / write
|
|
581
|
+
mode : str, optional
|
|
582
|
+
The read / write mode. The following modes are supported:
|
|
583
|
+
* 'r' open for reading (default)
|
|
584
|
+
* 'w' open for writing, truncating the file first
|
|
585
|
+
* 'x' create a new file and open it for writing
|
|
586
|
+
The data type can be specified by adding one of the following:
|
|
587
|
+
* 'b' binary mode
|
|
588
|
+
* 't' text mode (default)
|
|
589
|
+
encoding : str, optional
|
|
590
|
+
The string encoding to use for text
|
|
591
|
+
|
|
592
|
+
Returns
|
|
593
|
+
-------
|
|
594
|
+
FilesObjectBytesReader - 'rb' or 'b' mode
|
|
595
|
+
FilesObjectBytesWriter - 'wb' or 'xb' mode
|
|
596
|
+
FilesObjectTextReader - 'r' or 'rt' mode
|
|
597
|
+
FilesObjectTextWriter - 'w', 'x', 'wt' or 'xt' mode
|
|
598
|
+
|
|
599
|
+
"""
|
|
600
|
+
if '+' in mode or 'a' in mode:
|
|
601
|
+
raise ManagementError(msg='modifying an existing file is not supported')
|
|
602
|
+
|
|
603
|
+
if 'w' in mode or 'x' in mode:
|
|
604
|
+
exists = self.exists(path)
|
|
605
|
+
if exists:
|
|
606
|
+
if 'x' in mode:
|
|
607
|
+
raise FileExistsError(f'file path already exists: {path}')
|
|
608
|
+
self.remove(path)
|
|
609
|
+
if 'b' in mode:
|
|
610
|
+
return FilesObjectBytesWriter(b'', self, path)
|
|
611
|
+
return FilesObjectTextWriter('', self, path)
|
|
612
|
+
|
|
613
|
+
if 'r' in mode:
|
|
614
|
+
content = self.download_file(path)
|
|
615
|
+
if isinstance(content, bytes):
|
|
616
|
+
if 'b' in mode:
|
|
617
|
+
return FilesObjectBytesReader(content)
|
|
618
|
+
encoding = 'utf-8' if encoding is None else encoding
|
|
619
|
+
return FilesObjectTextReader(content.decode(encoding))
|
|
620
|
+
|
|
621
|
+
if isinstance(content, str):
|
|
622
|
+
return FilesObjectTextReader(content)
|
|
623
|
+
|
|
624
|
+
raise ValueError(f'unrecognized file content type: {type(content)}')
|
|
625
|
+
|
|
626
|
+
raise ValueError(f'must have one of create/read/write mode specified: {mode}')
|
|
627
|
+
|
|
628
|
+
def upload_file(
|
|
629
|
+
self,
|
|
630
|
+
local_path: Union[PathLike, io.IOBase],
|
|
631
|
+
path: PathLike,
|
|
632
|
+
*,
|
|
633
|
+
overwrite: bool = False,
|
|
634
|
+
) -> FilesObject:
|
|
635
|
+
"""
|
|
636
|
+
Upload a local file.
|
|
637
|
+
|
|
638
|
+
Parameters
|
|
639
|
+
----------
|
|
640
|
+
local_path : Path or str or file-like
|
|
641
|
+
Path to the local file or an open file object
|
|
642
|
+
path : Path or str
|
|
643
|
+
Path to the file
|
|
644
|
+
overwrite : bool, optional
|
|
645
|
+
Should the ``path`` be overwritten if it exists already?
|
|
646
|
+
|
|
647
|
+
"""
|
|
648
|
+
if isinstance(local_path, io.IOBase):
|
|
649
|
+
pass
|
|
650
|
+
elif not os.path.isfile(local_path):
|
|
651
|
+
raise IsADirectoryError(f'local path is not a file: {local_path}')
|
|
652
|
+
|
|
653
|
+
if self.exists(path):
|
|
654
|
+
if not overwrite:
|
|
655
|
+
raise OSError(f'file path already exists: {path}')
|
|
656
|
+
|
|
657
|
+
self.remove(path)
|
|
658
|
+
|
|
659
|
+
if isinstance(local_path, io.IOBase):
|
|
660
|
+
return self._upload(local_path, path, overwrite=overwrite)
|
|
661
|
+
|
|
662
|
+
return self._upload(open(local_path, 'rb'), path, overwrite=overwrite)
|
|
663
|
+
|
|
664
|
+
def upload_folder(
|
|
665
|
+
self,
|
|
666
|
+
local_path: PathLike,
|
|
667
|
+
path: PathLike,
|
|
668
|
+
*,
|
|
669
|
+
overwrite: bool = False,
|
|
670
|
+
recursive: bool = True,
|
|
671
|
+
include_root: bool = False,
|
|
672
|
+
ignore: Optional[Union[PathLike, List[PathLike]]] = None,
|
|
673
|
+
) -> FilesObject:
|
|
674
|
+
"""
|
|
675
|
+
Upload a folder recursively.
|
|
676
|
+
|
|
677
|
+
Only the contents of the folder are uploaded. To include the
|
|
678
|
+
folder name itself in the target path use ``include_root=True``.
|
|
679
|
+
|
|
680
|
+
Parameters
|
|
681
|
+
----------
|
|
682
|
+
local_path : Path or str
|
|
683
|
+
Local directory to upload
|
|
684
|
+
path : Path or str
|
|
685
|
+
Path of folder to upload to
|
|
686
|
+
overwrite : bool, optional
|
|
687
|
+
If a file already exists, should it be overwritten?
|
|
688
|
+
recursive : bool, optional
|
|
689
|
+
Should nested folders be uploaded?
|
|
690
|
+
include_root : bool, optional
|
|
691
|
+
Should the local root folder itself be uploaded as the top folder?
|
|
692
|
+
ignore : Path or str or List[Path] or List[str], optional
|
|
693
|
+
Glob patterns of files to ignore, for example, '**/*.pyc` will
|
|
694
|
+
ignore all '*.pyc' files in the directory tree
|
|
695
|
+
|
|
696
|
+
"""
|
|
697
|
+
if not os.path.isdir(local_path):
|
|
698
|
+
raise NotADirectoryError(f'local path is not a directory: {local_path}')
|
|
699
|
+
|
|
700
|
+
if not path:
|
|
701
|
+
path = local_path
|
|
702
|
+
|
|
703
|
+
ignore_files = set()
|
|
704
|
+
if ignore:
|
|
705
|
+
if isinstance(ignore, list):
|
|
706
|
+
for item in ignore:
|
|
707
|
+
ignore_files.update(glob.glob(str(item), recursive=recursive))
|
|
708
|
+
else:
|
|
709
|
+
ignore_files.update(glob.glob(str(ignore), recursive=recursive))
|
|
710
|
+
|
|
711
|
+
for dir_path, _, files in os.walk(str(local_path)):
|
|
712
|
+
for fname in files:
|
|
713
|
+
if ignore_files and fname in ignore_files:
|
|
714
|
+
continue
|
|
715
|
+
|
|
716
|
+
local_file_path = os.path.join(dir_path, fname)
|
|
717
|
+
remote_path = os.path.join(
|
|
718
|
+
path,
|
|
719
|
+
local_file_path.lstrip(str(local_path)),
|
|
720
|
+
)
|
|
721
|
+
self.upload_file(
|
|
722
|
+
local_path=local_file_path,
|
|
723
|
+
path=remote_path,
|
|
724
|
+
overwrite=overwrite,
|
|
725
|
+
)
|
|
726
|
+
return self.info(path)
|
|
727
|
+
|
|
728
|
+
def _upload(
|
|
729
|
+
self,
|
|
730
|
+
content: Union[str, bytes, io.IOBase],
|
|
731
|
+
path: PathLike,
|
|
732
|
+
*,
|
|
733
|
+
overwrite: bool = False,
|
|
734
|
+
) -> FilesObject:
|
|
735
|
+
"""
|
|
736
|
+
Upload content to a file.
|
|
737
|
+
|
|
738
|
+
Parameters
|
|
739
|
+
----------
|
|
740
|
+
content : str or bytes or file-like
|
|
741
|
+
Content to upload
|
|
742
|
+
path : Path or str
|
|
743
|
+
Path to the file
|
|
744
|
+
overwrite : bool, optional
|
|
745
|
+
Should the ``path`` be overwritten if it exists already?
|
|
746
|
+
|
|
747
|
+
"""
|
|
748
|
+
if self.exists(path):
|
|
749
|
+
if not overwrite:
|
|
750
|
+
raise OSError(f'file path already exists: {path}')
|
|
751
|
+
self.remove(path)
|
|
752
|
+
|
|
753
|
+
self._manager._put(
|
|
754
|
+
f'files/fs/{self._location}/{path}',
|
|
755
|
+
files={'file': content},
|
|
756
|
+
headers={'Content-Type': None},
|
|
757
|
+
)
|
|
758
|
+
|
|
759
|
+
return self.info(path)
|
|
760
|
+
|
|
761
|
+
def mkdir(self, path: PathLike, overwrite: bool = False) -> FilesObject:
|
|
762
|
+
"""
|
|
763
|
+
Make a directory in the file space.
|
|
764
|
+
|
|
765
|
+
Parameters
|
|
766
|
+
----------
|
|
767
|
+
path : Path or str
|
|
768
|
+
Path of the folder to create
|
|
769
|
+
overwrite : bool, optional
|
|
770
|
+
Should the file path be overwritten if it exists already?
|
|
771
|
+
|
|
772
|
+
Returns
|
|
773
|
+
-------
|
|
774
|
+
FilesObject
|
|
775
|
+
|
|
776
|
+
"""
|
|
777
|
+
raise ManagementError(
|
|
778
|
+
msg='Operation not supported: directories are currently not allowed '
|
|
779
|
+
'in Files API',
|
|
780
|
+
)
|
|
781
|
+
|
|
782
|
+
mkdirs = mkdir
|
|
783
|
+
|
|
784
|
+
def rename(
|
|
785
|
+
self,
|
|
786
|
+
old_path: PathLike,
|
|
787
|
+
new_path: PathLike,
|
|
788
|
+
*,
|
|
789
|
+
overwrite: bool = False,
|
|
790
|
+
) -> FilesObject:
|
|
791
|
+
"""
|
|
792
|
+
Move the file to a new location.
|
|
793
|
+
|
|
794
|
+
Parameters
|
|
795
|
+
-----------
|
|
796
|
+
old_path : Path or str
|
|
797
|
+
Original location of the path
|
|
798
|
+
new_path : Path or str
|
|
799
|
+
New location of the path
|
|
800
|
+
overwrite : bool, optional
|
|
801
|
+
Should the ``new_path`` be overwritten if it exists already?
|
|
802
|
+
|
|
803
|
+
"""
|
|
804
|
+
if not self.exists(old_path):
|
|
805
|
+
raise OSError(f'file path does not exist: {old_path}')
|
|
806
|
+
|
|
807
|
+
if str(old_path).endswith('/') or str(new_path).endswith('/'):
|
|
808
|
+
raise ManagementError(
|
|
809
|
+
msg='Operation not supported: directories are currently not allowed '
|
|
810
|
+
'in Files API',
|
|
811
|
+
)
|
|
812
|
+
|
|
813
|
+
if self.exists(new_path):
|
|
814
|
+
if not overwrite:
|
|
815
|
+
raise OSError(f'file path already exists: {new_path}')
|
|
816
|
+
|
|
817
|
+
self.remove(new_path)
|
|
818
|
+
|
|
819
|
+
self._manager._patch(
|
|
820
|
+
f'files/fs/{self._location}/{old_path}',
|
|
821
|
+
json=dict(newPath=new_path),
|
|
822
|
+
)
|
|
823
|
+
|
|
824
|
+
return self.info(new_path)
|
|
825
|
+
|
|
826
|
+
def info(self, path: PathLike) -> FilesObject:
|
|
827
|
+
"""
|
|
828
|
+
Return information about a file location.
|
|
829
|
+
|
|
830
|
+
Parameters
|
|
831
|
+
----------
|
|
832
|
+
path : Path or str
|
|
833
|
+
Path to the file
|
|
834
|
+
|
|
835
|
+
Returns
|
|
836
|
+
-------
|
|
837
|
+
FilesObject
|
|
838
|
+
|
|
839
|
+
"""
|
|
840
|
+
res = self._manager._get(
|
|
841
|
+
re.sub(r'/+$', r'/', f'files/fs/{self._location}/{path}'),
|
|
842
|
+
params=dict(metadata=1),
|
|
843
|
+
).json()
|
|
844
|
+
|
|
845
|
+
return FilesObject.from_dict(res, self)
|
|
846
|
+
|
|
847
|
+
def exists(self, path: PathLike) -> bool:
|
|
848
|
+
"""
|
|
849
|
+
Does the given file path exist?
|
|
850
|
+
|
|
851
|
+
Parameters
|
|
852
|
+
----------
|
|
853
|
+
path : Path or str
|
|
854
|
+
Path to file object
|
|
855
|
+
|
|
856
|
+
Returns
|
|
857
|
+
-------
|
|
858
|
+
bool
|
|
859
|
+
|
|
860
|
+
"""
|
|
861
|
+
try:
|
|
862
|
+
self.info(path)
|
|
863
|
+
return True
|
|
864
|
+
except ManagementError as exc:
|
|
865
|
+
if exc.errno == 404:
|
|
866
|
+
return False
|
|
867
|
+
raise
|
|
868
|
+
|
|
869
|
+
def is_dir(self, path: PathLike) -> bool:
|
|
870
|
+
"""
|
|
871
|
+
Is the given file path a directory?
|
|
872
|
+
|
|
873
|
+
Parameters
|
|
874
|
+
----------
|
|
875
|
+
path : Path or str
|
|
876
|
+
Path to file object
|
|
877
|
+
|
|
878
|
+
Returns
|
|
879
|
+
-------
|
|
880
|
+
bool
|
|
881
|
+
|
|
882
|
+
"""
|
|
883
|
+
try:
|
|
884
|
+
return self.info(path).type == 'directory'
|
|
885
|
+
except ManagementError as exc:
|
|
886
|
+
if exc.errno == 404:
|
|
887
|
+
return False
|
|
888
|
+
raise
|
|
889
|
+
|
|
890
|
+
def is_file(self, path: PathLike) -> bool:
|
|
891
|
+
"""
|
|
892
|
+
Is the given file path a file?
|
|
893
|
+
|
|
894
|
+
Parameters
|
|
895
|
+
----------
|
|
896
|
+
path : Path or str
|
|
897
|
+
Path to file object
|
|
898
|
+
|
|
899
|
+
Returns
|
|
900
|
+
-------
|
|
901
|
+
bool
|
|
902
|
+
|
|
903
|
+
"""
|
|
904
|
+
try:
|
|
905
|
+
return self.info(path).type != 'directory'
|
|
906
|
+
except ManagementError as exc:
|
|
907
|
+
if exc.errno == 404:
|
|
908
|
+
return False
|
|
909
|
+
raise
|
|
910
|
+
|
|
911
|
+
def _listdir(self, path: PathLike, *, recursive: bool = False) -> List[str]:
|
|
912
|
+
"""
|
|
913
|
+
Return the names of files in a directory.
|
|
914
|
+
|
|
915
|
+
Parameters
|
|
916
|
+
----------
|
|
917
|
+
path : Path or str
|
|
918
|
+
Path to the folder
|
|
919
|
+
recursive : bool, optional
|
|
920
|
+
Should folders be listed recursively?
|
|
921
|
+
|
|
922
|
+
"""
|
|
923
|
+
res = self._manager._get(
|
|
924
|
+
f'files/fs/{self._location}/{path}',
|
|
925
|
+
).json()
|
|
926
|
+
|
|
927
|
+
if recursive:
|
|
928
|
+
out = []
|
|
929
|
+
for item in res['content'] or []:
|
|
930
|
+
out.append(item['path'])
|
|
931
|
+
if item['type'] == 'directory':
|
|
932
|
+
out.extend(self._listdir(item['path'], recursive=recursive))
|
|
933
|
+
return out
|
|
934
|
+
|
|
935
|
+
return [x['path'] for x in res['content'] or []]
|
|
936
|
+
|
|
937
|
+
def listdir(
|
|
938
|
+
self,
|
|
939
|
+
path: PathLike = '/',
|
|
940
|
+
*,
|
|
941
|
+
recursive: bool = False,
|
|
942
|
+
) -> List[str]:
|
|
943
|
+
"""
|
|
944
|
+
List the files / folders at the given path.
|
|
945
|
+
|
|
946
|
+
Parameters
|
|
947
|
+
----------
|
|
948
|
+
path : Path or str, optional
|
|
949
|
+
Path to the file location
|
|
950
|
+
|
|
951
|
+
Returns
|
|
952
|
+
-------
|
|
953
|
+
List[str]
|
|
954
|
+
|
|
955
|
+
"""
|
|
956
|
+
path = re.sub(r'^(\./|/)+', r'', str(path))
|
|
957
|
+
path = re.sub(r'/+$', r'', path) + '/'
|
|
958
|
+
|
|
959
|
+
if not self.is_dir(path):
|
|
960
|
+
raise NotADirectoryError(f'path is not a directory: {path}')
|
|
961
|
+
|
|
962
|
+
out = self._listdir(path, recursive=recursive)
|
|
963
|
+
if path != '/':
|
|
964
|
+
path_n = len(path.split('/')) - 1
|
|
965
|
+
out = ['/'.join(x.split('/')[path_n:]) for x in out]
|
|
966
|
+
return out
|
|
967
|
+
|
|
968
|
+
def download_file(
|
|
969
|
+
self,
|
|
970
|
+
path: PathLike,
|
|
971
|
+
local_path: Optional[PathLike] = None,
|
|
972
|
+
*,
|
|
973
|
+
overwrite: bool = False,
|
|
974
|
+
encoding: Optional[str] = None,
|
|
975
|
+
) -> Optional[Union[bytes, str]]:
|
|
976
|
+
"""
|
|
977
|
+
Download the content of a file path.
|
|
978
|
+
|
|
979
|
+
Parameters
|
|
980
|
+
----------
|
|
981
|
+
path : Path or str
|
|
982
|
+
Path to the file
|
|
983
|
+
local_path : Path or str
|
|
984
|
+
Path to local file target location
|
|
985
|
+
overwrite : bool, optional
|
|
986
|
+
Should an existing file be overwritten if it exists?
|
|
987
|
+
encoding : str, optional
|
|
988
|
+
Encoding used to convert the resulting data
|
|
989
|
+
|
|
990
|
+
Returns
|
|
991
|
+
-------
|
|
992
|
+
bytes or str - ``local_path`` is None
|
|
993
|
+
None - ``local_path`` is a Path or str
|
|
994
|
+
|
|
995
|
+
"""
|
|
996
|
+
if local_path is not None and not overwrite and os.path.exists(local_path):
|
|
997
|
+
raise OSError('target file already exists; use overwrite=True to replace')
|
|
998
|
+
if self.is_dir(path):
|
|
999
|
+
raise IsADirectoryError(f'file path is a directory: {path}')
|
|
1000
|
+
|
|
1001
|
+
out = self._manager._get(
|
|
1002
|
+
f'files/fs/{self._location}/{path}',
|
|
1003
|
+
).content
|
|
1004
|
+
|
|
1005
|
+
if local_path is not None:
|
|
1006
|
+
with open(local_path, 'wb') as outfile:
|
|
1007
|
+
outfile.write(out)
|
|
1008
|
+
return None
|
|
1009
|
+
|
|
1010
|
+
if encoding:
|
|
1011
|
+
return out.decode(encoding)
|
|
1012
|
+
|
|
1013
|
+
return out
|
|
1014
|
+
|
|
1015
|
+
def download_folder(
|
|
1016
|
+
self,
|
|
1017
|
+
path: PathLike,
|
|
1018
|
+
local_path: PathLike = '.',
|
|
1019
|
+
*,
|
|
1020
|
+
overwrite: bool = False,
|
|
1021
|
+
) -> None:
|
|
1022
|
+
"""
|
|
1023
|
+
Download a FileSpace folder to a local directory.
|
|
1024
|
+
|
|
1025
|
+
Parameters
|
|
1026
|
+
----------
|
|
1027
|
+
path : Path or str
|
|
1028
|
+
Directory path
|
|
1029
|
+
local_path : Path or str
|
|
1030
|
+
Path to local directory target location
|
|
1031
|
+
overwrite : bool, optional
|
|
1032
|
+
Should an existing directory / files be overwritten if they exist?
|
|
1033
|
+
|
|
1034
|
+
"""
|
|
1035
|
+
|
|
1036
|
+
if local_path is not None and not overwrite and os.path.exists(local_path):
|
|
1037
|
+
raise OSError('target path already exists; use overwrite=True to replace')
|
|
1038
|
+
|
|
1039
|
+
if not self.is_dir(path):
|
|
1040
|
+
raise NotADirectoryError(f'path is not a directory: {path}')
|
|
1041
|
+
|
|
1042
|
+
files = self.listdir(path, recursive=True)
|
|
1043
|
+
for f in files:
|
|
1044
|
+
remote_path = os.path.join(path, f)
|
|
1045
|
+
if self.is_dir(remote_path):
|
|
1046
|
+
continue
|
|
1047
|
+
target = os.path.normpath(os.path.join(local_path, f))
|
|
1048
|
+
os.makedirs(os.path.dirname(target), exist_ok=True)
|
|
1049
|
+
self.download_file(remote_path, target, overwrite=overwrite)
|
|
1050
|
+
|
|
1051
|
+
def remove(self, path: PathLike) -> None:
|
|
1052
|
+
"""
|
|
1053
|
+
Delete a file location.
|
|
1054
|
+
|
|
1055
|
+
Parameters
|
|
1056
|
+
----------
|
|
1057
|
+
path : Path or str
|
|
1058
|
+
Path to the location
|
|
1059
|
+
|
|
1060
|
+
"""
|
|
1061
|
+
if self.is_dir(path):
|
|
1062
|
+
raise IsADirectoryError('file path is a directory')
|
|
1063
|
+
|
|
1064
|
+
self._manager._delete(f'files/fs/{self._location}/{path}')
|
|
1065
|
+
|
|
1066
|
+
def removedirs(self, path: PathLike) -> None:
|
|
1067
|
+
"""
|
|
1068
|
+
Delete a folder recursively.
|
|
1069
|
+
|
|
1070
|
+
Parameters
|
|
1071
|
+
----------
|
|
1072
|
+
path : Path or str
|
|
1073
|
+
Path to the file location
|
|
1074
|
+
|
|
1075
|
+
"""
|
|
1076
|
+
if not self.is_dir(path):
|
|
1077
|
+
raise NotADirectoryError('path is not a directory')
|
|
1078
|
+
|
|
1079
|
+
self._manager._delete(f'files/fs/{self._location}/{path}')
|
|
1080
|
+
|
|
1081
|
+
def rmdir(self, path: PathLike) -> None:
|
|
1082
|
+
"""
|
|
1083
|
+
Delete a folder.
|
|
1084
|
+
|
|
1085
|
+
Parameters
|
|
1086
|
+
----------
|
|
1087
|
+
path : Path or str
|
|
1088
|
+
Path to the file location
|
|
1089
|
+
|
|
1090
|
+
"""
|
|
1091
|
+
raise ManagementError(
|
|
1092
|
+
msg='Operation not supported: directories are currently not allowed '
|
|
1093
|
+
'in Files API',
|
|
1094
|
+
)
|
|
1095
|
+
|
|
1096
|
+
def __str__(self) -> str:
|
|
1097
|
+
"""Return string representation."""
|
|
1098
|
+
return vars_to_str(self)
|
|
1099
|
+
|
|
1100
|
+
def __repr__(self) -> str:
|
|
1101
|
+
"""Return string representation."""
|
|
1102
|
+
return str(self)
|