singlestoredb 1.16.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (183) hide show
  1. singlestoredb/__init__.py +75 -0
  2. singlestoredb/ai/__init__.py +2 -0
  3. singlestoredb/ai/chat.py +139 -0
  4. singlestoredb/ai/embeddings.py +128 -0
  5. singlestoredb/alchemy/__init__.py +90 -0
  6. singlestoredb/apps/__init__.py +3 -0
  7. singlestoredb/apps/_cloud_functions.py +90 -0
  8. singlestoredb/apps/_config.py +72 -0
  9. singlestoredb/apps/_connection_info.py +18 -0
  10. singlestoredb/apps/_dashboards.py +47 -0
  11. singlestoredb/apps/_process.py +32 -0
  12. singlestoredb/apps/_python_udfs.py +100 -0
  13. singlestoredb/apps/_stdout_supress.py +30 -0
  14. singlestoredb/apps/_uvicorn_util.py +36 -0
  15. singlestoredb/auth.py +245 -0
  16. singlestoredb/config.py +484 -0
  17. singlestoredb/connection.py +1487 -0
  18. singlestoredb/converters.py +950 -0
  19. singlestoredb/docstring/__init__.py +33 -0
  20. singlestoredb/docstring/attrdoc.py +126 -0
  21. singlestoredb/docstring/common.py +230 -0
  22. singlestoredb/docstring/epydoc.py +267 -0
  23. singlestoredb/docstring/google.py +412 -0
  24. singlestoredb/docstring/numpydoc.py +562 -0
  25. singlestoredb/docstring/parser.py +100 -0
  26. singlestoredb/docstring/py.typed +1 -0
  27. singlestoredb/docstring/rest.py +256 -0
  28. singlestoredb/docstring/tests/__init__.py +1 -0
  29. singlestoredb/docstring/tests/_pydoctor.py +21 -0
  30. singlestoredb/docstring/tests/test_epydoc.py +729 -0
  31. singlestoredb/docstring/tests/test_google.py +1007 -0
  32. singlestoredb/docstring/tests/test_numpydoc.py +1100 -0
  33. singlestoredb/docstring/tests/test_parse_from_object.py +109 -0
  34. singlestoredb/docstring/tests/test_parser.py +248 -0
  35. singlestoredb/docstring/tests/test_rest.py +547 -0
  36. singlestoredb/docstring/tests/test_util.py +70 -0
  37. singlestoredb/docstring/util.py +141 -0
  38. singlestoredb/exceptions.py +120 -0
  39. singlestoredb/functions/__init__.py +16 -0
  40. singlestoredb/functions/decorator.py +201 -0
  41. singlestoredb/functions/dtypes.py +1793 -0
  42. singlestoredb/functions/ext/__init__.py +1 -0
  43. singlestoredb/functions/ext/arrow.py +375 -0
  44. singlestoredb/functions/ext/asgi.py +2133 -0
  45. singlestoredb/functions/ext/json.py +420 -0
  46. singlestoredb/functions/ext/mmap.py +413 -0
  47. singlestoredb/functions/ext/rowdat_1.py +724 -0
  48. singlestoredb/functions/ext/timer.py +89 -0
  49. singlestoredb/functions/ext/utils.py +218 -0
  50. singlestoredb/functions/signature.py +1578 -0
  51. singlestoredb/functions/typing/__init__.py +41 -0
  52. singlestoredb/functions/typing/numpy.py +20 -0
  53. singlestoredb/functions/typing/pandas.py +2 -0
  54. singlestoredb/functions/typing/polars.py +2 -0
  55. singlestoredb/functions/typing/pyarrow.py +2 -0
  56. singlestoredb/functions/utils.py +421 -0
  57. singlestoredb/fusion/__init__.py +11 -0
  58. singlestoredb/fusion/graphql.py +213 -0
  59. singlestoredb/fusion/handler.py +916 -0
  60. singlestoredb/fusion/handlers/__init__.py +0 -0
  61. singlestoredb/fusion/handlers/export.py +525 -0
  62. singlestoredb/fusion/handlers/files.py +690 -0
  63. singlestoredb/fusion/handlers/job.py +660 -0
  64. singlestoredb/fusion/handlers/models.py +250 -0
  65. singlestoredb/fusion/handlers/stage.py +502 -0
  66. singlestoredb/fusion/handlers/utils.py +324 -0
  67. singlestoredb/fusion/handlers/workspace.py +956 -0
  68. singlestoredb/fusion/registry.py +249 -0
  69. singlestoredb/fusion/result.py +399 -0
  70. singlestoredb/http/__init__.py +27 -0
  71. singlestoredb/http/connection.py +1267 -0
  72. singlestoredb/magics/__init__.py +34 -0
  73. singlestoredb/magics/run_personal.py +137 -0
  74. singlestoredb/magics/run_shared.py +134 -0
  75. singlestoredb/management/__init__.py +9 -0
  76. singlestoredb/management/billing_usage.py +148 -0
  77. singlestoredb/management/cluster.py +462 -0
  78. singlestoredb/management/export.py +295 -0
  79. singlestoredb/management/files.py +1102 -0
  80. singlestoredb/management/inference_api.py +105 -0
  81. singlestoredb/management/job.py +887 -0
  82. singlestoredb/management/manager.py +373 -0
  83. singlestoredb/management/organization.py +226 -0
  84. singlestoredb/management/region.py +169 -0
  85. singlestoredb/management/utils.py +423 -0
  86. singlestoredb/management/workspace.py +1927 -0
  87. singlestoredb/mysql/__init__.py +177 -0
  88. singlestoredb/mysql/_auth.py +298 -0
  89. singlestoredb/mysql/charset.py +214 -0
  90. singlestoredb/mysql/connection.py +2032 -0
  91. singlestoredb/mysql/constants/CLIENT.py +38 -0
  92. singlestoredb/mysql/constants/COMMAND.py +32 -0
  93. singlestoredb/mysql/constants/CR.py +78 -0
  94. singlestoredb/mysql/constants/ER.py +474 -0
  95. singlestoredb/mysql/constants/EXTENDED_TYPE.py +3 -0
  96. singlestoredb/mysql/constants/FIELD_TYPE.py +48 -0
  97. singlestoredb/mysql/constants/FLAG.py +15 -0
  98. singlestoredb/mysql/constants/SERVER_STATUS.py +10 -0
  99. singlestoredb/mysql/constants/VECTOR_TYPE.py +6 -0
  100. singlestoredb/mysql/constants/__init__.py +0 -0
  101. singlestoredb/mysql/converters.py +271 -0
  102. singlestoredb/mysql/cursors.py +896 -0
  103. singlestoredb/mysql/err.py +92 -0
  104. singlestoredb/mysql/optionfile.py +20 -0
  105. singlestoredb/mysql/protocol.py +450 -0
  106. singlestoredb/mysql/tests/__init__.py +19 -0
  107. singlestoredb/mysql/tests/base.py +126 -0
  108. singlestoredb/mysql/tests/conftest.py +37 -0
  109. singlestoredb/mysql/tests/test_DictCursor.py +132 -0
  110. singlestoredb/mysql/tests/test_SSCursor.py +141 -0
  111. singlestoredb/mysql/tests/test_basic.py +452 -0
  112. singlestoredb/mysql/tests/test_connection.py +851 -0
  113. singlestoredb/mysql/tests/test_converters.py +58 -0
  114. singlestoredb/mysql/tests/test_cursor.py +141 -0
  115. singlestoredb/mysql/tests/test_err.py +16 -0
  116. singlestoredb/mysql/tests/test_issues.py +514 -0
  117. singlestoredb/mysql/tests/test_load_local.py +75 -0
  118. singlestoredb/mysql/tests/test_nextset.py +88 -0
  119. singlestoredb/mysql/tests/test_optionfile.py +27 -0
  120. singlestoredb/mysql/tests/thirdparty/__init__.py +6 -0
  121. singlestoredb/mysql/tests/thirdparty/test_MySQLdb/__init__.py +9 -0
  122. singlestoredb/mysql/tests/thirdparty/test_MySQLdb/capabilities.py +323 -0
  123. singlestoredb/mysql/tests/thirdparty/test_MySQLdb/dbapi20.py +865 -0
  124. singlestoredb/mysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_capabilities.py +110 -0
  125. singlestoredb/mysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_dbapi20.py +224 -0
  126. singlestoredb/mysql/tests/thirdparty/test_MySQLdb/test_MySQLdb_nonstandard.py +101 -0
  127. singlestoredb/mysql/times.py +23 -0
  128. singlestoredb/notebook/__init__.py +16 -0
  129. singlestoredb/notebook/_objects.py +213 -0
  130. singlestoredb/notebook/_portal.py +352 -0
  131. singlestoredb/py.typed +0 -0
  132. singlestoredb/pytest.py +352 -0
  133. singlestoredb/server/__init__.py +0 -0
  134. singlestoredb/server/docker.py +452 -0
  135. singlestoredb/server/free_tier.py +267 -0
  136. singlestoredb/tests/__init__.py +0 -0
  137. singlestoredb/tests/alltypes.sql +307 -0
  138. singlestoredb/tests/alltypes_no_nulls.sql +208 -0
  139. singlestoredb/tests/empty.sql +0 -0
  140. singlestoredb/tests/ext_funcs/__init__.py +702 -0
  141. singlestoredb/tests/local_infile.csv +3 -0
  142. singlestoredb/tests/test.ipynb +18 -0
  143. singlestoredb/tests/test.sql +680 -0
  144. singlestoredb/tests/test2.ipynb +18 -0
  145. singlestoredb/tests/test2.sql +1 -0
  146. singlestoredb/tests/test_basics.py +1332 -0
  147. singlestoredb/tests/test_config.py +318 -0
  148. singlestoredb/tests/test_connection.py +3103 -0
  149. singlestoredb/tests/test_dbapi.py +27 -0
  150. singlestoredb/tests/test_exceptions.py +45 -0
  151. singlestoredb/tests/test_ext_func.py +1472 -0
  152. singlestoredb/tests/test_ext_func_data.py +1101 -0
  153. singlestoredb/tests/test_fusion.py +1527 -0
  154. singlestoredb/tests/test_http.py +288 -0
  155. singlestoredb/tests/test_management.py +1599 -0
  156. singlestoredb/tests/test_plugin.py +33 -0
  157. singlestoredb/tests/test_results.py +171 -0
  158. singlestoredb/tests/test_types.py +132 -0
  159. singlestoredb/tests/test_udf.py +737 -0
  160. singlestoredb/tests/test_udf_returns.py +459 -0
  161. singlestoredb/tests/test_vectorstore.py +51 -0
  162. singlestoredb/tests/test_xdict.py +333 -0
  163. singlestoredb/tests/utils.py +141 -0
  164. singlestoredb/types.py +373 -0
  165. singlestoredb/utils/__init__.py +0 -0
  166. singlestoredb/utils/config.py +950 -0
  167. singlestoredb/utils/convert_rows.py +69 -0
  168. singlestoredb/utils/debug.py +13 -0
  169. singlestoredb/utils/dtypes.py +205 -0
  170. singlestoredb/utils/events.py +65 -0
  171. singlestoredb/utils/mogrify.py +151 -0
  172. singlestoredb/utils/results.py +585 -0
  173. singlestoredb/utils/xdict.py +425 -0
  174. singlestoredb/vectorstore.py +192 -0
  175. singlestoredb/warnings.py +5 -0
  176. singlestoredb-1.16.1.dist-info/METADATA +165 -0
  177. singlestoredb-1.16.1.dist-info/RECORD +183 -0
  178. singlestoredb-1.16.1.dist-info/WHEEL +5 -0
  179. singlestoredb-1.16.1.dist-info/entry_points.txt +2 -0
  180. singlestoredb-1.16.1.dist-info/licenses/LICENSE +201 -0
  181. singlestoredb-1.16.1.dist-info/top_level.txt +3 -0
  182. sqlx/__init__.py +4 -0
  183. sqlx/magic.py +113 -0
@@ -0,0 +1,1102 @@
1
+ #!/usr/bin/env python
2
+ """SingleStore Cloud Files Management."""
3
+ from __future__ import annotations
4
+
5
+ import datetime
6
+ import glob
7
+ import io
8
+ import os
9
+ import re
10
+ from abc import ABC
11
+ from abc import abstractmethod
12
+ from typing import Any
13
+ from typing import Dict
14
+ from typing import List
15
+ from typing import Optional
16
+ from typing import Union
17
+
18
+ from .. import config
19
+ from ..exceptions import ManagementError
20
+ from .manager import Manager
21
+ from .utils import PathLike
22
+ from .utils import to_datetime
23
+ from .utils import vars_to_str
24
+
25
+ PERSONAL_SPACE = 'personal'
26
+ SHARED_SPACE = 'shared'
27
+ MODELS_SPACE = 'models'
28
+
29
+
30
+ class FilesObject(object):
31
+ """
32
+ File / folder object.
33
+
34
+ It can belong to either a workspace stage or personal/shared space.
35
+
36
+ This object is not instantiated directly. It is used in the results
37
+ of various operations in ``WorkspaceGroup.stage``, ``FilesManager.personal_space``,
38
+ ``FilesManager.shared_space`` and ``FilesManager.models_space`` methods.
39
+
40
+ """
41
+
42
+ def __init__(
43
+ self,
44
+ name: str,
45
+ path: str,
46
+ size: int,
47
+ type: str,
48
+ format: str,
49
+ mimetype: str,
50
+ created: Optional[datetime.datetime],
51
+ last_modified: Optional[datetime.datetime],
52
+ writable: bool,
53
+ content: Optional[List[str]] = None,
54
+ ):
55
+ #: Name of file / folder
56
+ self.name = name
57
+
58
+ if type == 'directory':
59
+ path = re.sub(r'/*$', r'', str(path)) + '/'
60
+
61
+ #: Path of file / folder
62
+ self.path = path
63
+
64
+ #: Size of the object (in bytes)
65
+ self.size = size
66
+
67
+ #: Data type: file or directory
68
+ self.type = type
69
+
70
+ #: Data format
71
+ self.format = format
72
+
73
+ #: Mime type
74
+ self.mimetype = mimetype
75
+
76
+ #: Datetime the object was created
77
+ self.created_at = created
78
+
79
+ #: Datetime the object was modified last
80
+ self.last_modified_at = last_modified
81
+
82
+ #: Is the object writable?
83
+ self.writable = writable
84
+
85
+ #: Contents of a directory
86
+ self.content: List[str] = content or []
87
+
88
+ self._location: Optional[FileLocation] = None
89
+
90
+ @classmethod
91
+ def from_dict(
92
+ cls,
93
+ obj: Dict[str, Any],
94
+ location: FileLocation,
95
+ ) -> FilesObject:
96
+ """
97
+ Construct a FilesObject from a dictionary of values.
98
+
99
+ Parameters
100
+ ----------
101
+ obj : dict
102
+ Dictionary of values
103
+ location : FileLocation
104
+ FileLocation object to use as the parent
105
+
106
+ Returns
107
+ -------
108
+ :class:`FilesObject`
109
+
110
+ """
111
+ out = cls(
112
+ name=obj['name'],
113
+ path=obj['path'],
114
+ size=obj['size'],
115
+ type=obj['type'],
116
+ format=obj['format'],
117
+ mimetype=obj['mimetype'],
118
+ created=to_datetime(obj.get('created')),
119
+ last_modified=to_datetime(obj.get('last_modified')),
120
+ writable=bool(obj['writable']),
121
+ )
122
+ out._location = location
123
+ return out
124
+
125
+ def __str__(self) -> str:
126
+ """Return string representation."""
127
+ return vars_to_str(self)
128
+
129
+ def __repr__(self) -> str:
130
+ """Return string representation."""
131
+ return str(self)
132
+
133
+ def open(
134
+ self,
135
+ mode: str = 'r',
136
+ encoding: Optional[str] = None,
137
+ ) -> Union[io.StringIO, io.BytesIO]:
138
+ """
139
+ Open a file path for reading or writing.
140
+
141
+ Parameters
142
+ ----------
143
+ mode : str, optional
144
+ The read / write mode. The following modes are supported:
145
+ * 'r' open for reading (default)
146
+ * 'w' open for writing, truncating the file first
147
+ * 'x' create a new file and open it for writing
148
+ The data type can be specified by adding one of the following:
149
+ * 'b' binary mode
150
+ * 't' text mode (default)
151
+ encoding : str, optional
152
+ The string encoding to use for text
153
+
154
+ Returns
155
+ -------
156
+ FilesObjectBytesReader - 'rb' or 'b' mode
157
+ FilesObjectBytesWriter - 'wb' or 'xb' mode
158
+ FilesObjectTextReader - 'r' or 'rt' mode
159
+ FilesObjectTextWriter - 'w', 'x', 'wt' or 'xt' mode
160
+
161
+ """
162
+ if self._location is None:
163
+ raise ManagementError(
164
+ msg='No FileLocation object is associated with this object.',
165
+ )
166
+
167
+ if self.is_dir():
168
+ raise IsADirectoryError(
169
+ f'directories can not be read or written: {self.path}',
170
+ )
171
+
172
+ return self._location.open(self.path, mode=mode, encoding=encoding)
173
+
174
+ def download(
175
+ self,
176
+ local_path: Optional[PathLike] = None,
177
+ *,
178
+ overwrite: bool = False,
179
+ encoding: Optional[str] = None,
180
+ ) -> Optional[Union[bytes, str]]:
181
+ """
182
+ Download the content of a file path.
183
+
184
+ Parameters
185
+ ----------
186
+ local_path : Path or str
187
+ Path to local file target location
188
+ overwrite : bool, optional
189
+ Should an existing file be overwritten if it exists?
190
+ encoding : str, optional
191
+ Encoding used to convert the resulting data
192
+
193
+ Returns
194
+ -------
195
+ bytes or str or None
196
+
197
+ """
198
+ if self._location is None:
199
+ raise ManagementError(
200
+ msg='No FileLocation object is associated with this object.',
201
+ )
202
+
203
+ return self._location.download_file(
204
+ self.path, local_path=local_path,
205
+ overwrite=overwrite, encoding=encoding,
206
+ )
207
+
208
+ download_file = download
209
+
210
+ def remove(self) -> None:
211
+ """Delete the file."""
212
+ if self._location is None:
213
+ raise ManagementError(
214
+ msg='No FileLocation object is associated with this object.',
215
+ )
216
+
217
+ if self.type == 'directory':
218
+ raise IsADirectoryError(
219
+ f'path is a directory; use rmdir or removedirs {self.path}',
220
+ )
221
+
222
+ self._location.remove(self.path)
223
+
224
+ def rmdir(self) -> None:
225
+ """Delete the empty directory."""
226
+ if self._location is None:
227
+ raise ManagementError(
228
+ msg='No FileLocation object is associated with this object.',
229
+ )
230
+
231
+ if self.type != 'directory':
232
+ raise NotADirectoryError(
233
+ f'path is not a directory: {self.path}',
234
+ )
235
+
236
+ self._location.rmdir(self.path)
237
+
238
+ def removedirs(self) -> None:
239
+ """Delete the directory recursively."""
240
+ if self._location is None:
241
+ raise ManagementError(
242
+ msg='No FileLocation object is associated with this object.',
243
+ )
244
+
245
+ if self.type != 'directory':
246
+ raise NotADirectoryError(
247
+ f'path is not a directory: {self.path}',
248
+ )
249
+
250
+ self._location.removedirs(self.path)
251
+
252
+ def rename(self, new_path: PathLike, *, overwrite: bool = False) -> None:
253
+ """
254
+ Move the file to a new location.
255
+
256
+ Parameters
257
+ ----------
258
+ new_path : Path or str
259
+ The new location of the file
260
+ overwrite : bool, optional
261
+ Should path be overwritten if it already exists?
262
+
263
+ """
264
+ if self._location is None:
265
+ raise ManagementError(
266
+ msg='No FileLocation object is associated with this object.',
267
+ )
268
+ out = self._location.rename(self.path, new_path, overwrite=overwrite)
269
+ self.name = out.name
270
+ self.path = out.path
271
+ return None
272
+
273
+ def exists(self) -> bool:
274
+ """Does the file / folder exist?"""
275
+ if self._location is None:
276
+ raise ManagementError(
277
+ msg='No FileLocation object is associated with this object.',
278
+ )
279
+ return self._location.exists(self.path)
280
+
281
+ def is_dir(self) -> bool:
282
+ """Is the object a directory?"""
283
+ return self.type == 'directory'
284
+
285
+ def is_file(self) -> bool:
286
+ """Is the object a file?"""
287
+ return self.type != 'directory'
288
+
289
+ def abspath(self) -> str:
290
+ """Return the full path of the object."""
291
+ return str(self.path)
292
+
293
+ def basename(self) -> str:
294
+ """Return the basename of the object."""
295
+ return self.name
296
+
297
+ def dirname(self) -> str:
298
+ """Return the directory name of the object."""
299
+ return re.sub(r'/*$', r'', os.path.dirname(re.sub(r'/*$', r'', self.path))) + '/'
300
+
301
+ def getmtime(self) -> float:
302
+ """Return the last modified datetime as a UNIX timestamp."""
303
+ if self.last_modified_at is None:
304
+ return 0.0
305
+ return self.last_modified_at.timestamp()
306
+
307
+ def getctime(self) -> float:
308
+ """Return the creation datetime as a UNIX timestamp."""
309
+ if self.created_at is None:
310
+ return 0.0
311
+ return self.created_at.timestamp()
312
+
313
+
314
+ class FilesObjectTextWriter(io.StringIO):
315
+ """StringIO wrapper for writing to FileLocation."""
316
+
317
+ def __init__(self, buffer: Optional[str], location: FileLocation, path: PathLike):
318
+ self._location = location
319
+ self._path = path
320
+ super().__init__(buffer)
321
+
322
+ def close(self) -> None:
323
+ """Write the content to the path."""
324
+ self._location._upload(self.getvalue(), self._path)
325
+ super().close()
326
+
327
+
328
+ class FilesObjectTextReader(io.StringIO):
329
+ """StringIO wrapper for reading from FileLocation."""
330
+
331
+
332
+ class FilesObjectBytesWriter(io.BytesIO):
333
+ """BytesIO wrapper for writing to FileLocation."""
334
+
335
+ def __init__(self, buffer: bytes, location: FileLocation, path: PathLike):
336
+ self._location = location
337
+ self._path = path
338
+ super().__init__(buffer)
339
+
340
+ def close(self) -> None:
341
+ """Write the content to the file path."""
342
+ self._location._upload(self.getvalue(), self._path)
343
+ super().close()
344
+
345
+
346
+ class FilesObjectBytesReader(io.BytesIO):
347
+ """BytesIO wrapper for reading from FileLocation."""
348
+
349
+
350
+ class FileLocation(ABC):
351
+
352
+ @abstractmethod
353
+ def open(
354
+ self,
355
+ path: PathLike,
356
+ mode: str = 'r',
357
+ encoding: Optional[str] = None,
358
+ ) -> Union[io.StringIO, io.BytesIO]:
359
+ pass
360
+
361
+ @abstractmethod
362
+ def upload_file(
363
+ self,
364
+ local_path: Union[PathLike, io.IOBase],
365
+ path: PathLike,
366
+ *,
367
+ overwrite: bool = False,
368
+ ) -> FilesObject:
369
+ pass
370
+
371
+ @abstractmethod
372
+ def upload_folder(
373
+ self,
374
+ local_path: PathLike,
375
+ path: PathLike,
376
+ *,
377
+ overwrite: bool = False,
378
+ recursive: bool = True,
379
+ include_root: bool = False,
380
+ ignore: Optional[Union[PathLike, List[PathLike]]] = None,
381
+ ) -> FilesObject:
382
+ pass
383
+
384
+ @abstractmethod
385
+ def _upload(
386
+ self,
387
+ content: Union[str, bytes, io.IOBase],
388
+ path: PathLike,
389
+ *,
390
+ overwrite: bool = False,
391
+ ) -> FilesObject:
392
+ pass
393
+
394
+ @abstractmethod
395
+ def mkdir(self, path: PathLike, overwrite: bool = False) -> FilesObject:
396
+ pass
397
+
398
+ @abstractmethod
399
+ def rename(
400
+ self,
401
+ old_path: PathLike,
402
+ new_path: PathLike,
403
+ *,
404
+ overwrite: bool = False,
405
+ ) -> FilesObject:
406
+ pass
407
+
408
+ @abstractmethod
409
+ def info(self, path: PathLike) -> FilesObject:
410
+ pass
411
+
412
+ @abstractmethod
413
+ def exists(self, path: PathLike) -> bool:
414
+ pass
415
+
416
+ @abstractmethod
417
+ def is_dir(self, path: PathLike) -> bool:
418
+ pass
419
+
420
+ @abstractmethod
421
+ def is_file(self, path: PathLike) -> bool:
422
+ pass
423
+
424
+ @abstractmethod
425
+ def listdir(
426
+ self,
427
+ path: PathLike = '/',
428
+ *,
429
+ recursive: bool = False,
430
+ ) -> List[str]:
431
+ pass
432
+
433
+ @abstractmethod
434
+ def download_file(
435
+ self,
436
+ path: PathLike,
437
+ local_path: Optional[PathLike] = None,
438
+ *,
439
+ overwrite: bool = False,
440
+ encoding: Optional[str] = None,
441
+ ) -> Optional[Union[bytes, str]]:
442
+ pass
443
+
444
+ @abstractmethod
445
+ def download_folder(
446
+ self,
447
+ path: PathLike,
448
+ local_path: PathLike = '.',
449
+ *,
450
+ overwrite: bool = False,
451
+ ) -> None:
452
+ pass
453
+
454
+ @abstractmethod
455
+ def remove(self, path: PathLike) -> None:
456
+ pass
457
+
458
+ @abstractmethod
459
+ def removedirs(self, path: PathLike) -> None:
460
+ pass
461
+
462
+ @abstractmethod
463
+ def rmdir(self, path: PathLike) -> None:
464
+ pass
465
+
466
+ @abstractmethod
467
+ def __str__(self) -> str:
468
+ pass
469
+
470
+ @abstractmethod
471
+ def __repr__(self) -> str:
472
+ pass
473
+
474
+
475
+ class FilesManager(Manager):
476
+ """
477
+ SingleStoreDB files manager.
478
+
479
+ This class should be instantiated using :func:`singlestoredb.manage_files`.
480
+
481
+ Parameters
482
+ ----------
483
+ access_token : str, optional
484
+ The API key or other access token for the files management API
485
+ version : str, optional
486
+ Version of the API to use
487
+ base_url : str, optional
488
+ Base URL of the files management API
489
+
490
+ See Also
491
+ --------
492
+ :func:`singlestoredb.manage_files`
493
+
494
+ """
495
+
496
+ #: Management API version if none is specified.
497
+ default_version = config.get_option('management.version') or 'v1'
498
+
499
+ #: Base URL if none is specified.
500
+ default_base_url = config.get_option('management.base_url') \
501
+ or 'https://api.singlestore.com'
502
+
503
+ #: Object type
504
+ obj_type = 'file'
505
+
506
+ @property
507
+ def personal_space(self) -> FileSpace:
508
+ """Return the personal file space."""
509
+ return FileSpace(PERSONAL_SPACE, self)
510
+
511
+ @property
512
+ def shared_space(self) -> FileSpace:
513
+ """Return the shared file space."""
514
+ return FileSpace(SHARED_SPACE, self)
515
+
516
+ @property
517
+ def models_space(self) -> FileSpace:
518
+ """Return the models file space."""
519
+ return FileSpace(MODELS_SPACE, self)
520
+
521
+
522
+ def manage_files(
523
+ access_token: Optional[str] = None,
524
+ version: Optional[str] = None,
525
+ base_url: Optional[str] = None,
526
+ *,
527
+ organization_id: Optional[str] = None,
528
+ ) -> FilesManager:
529
+ """
530
+ Retrieve a SingleStoreDB files manager.
531
+
532
+ Parameters
533
+ ----------
534
+ access_token : str, optional
535
+ The API key or other access token for the files management API
536
+ version : str, optional
537
+ Version of the API to use
538
+ base_url : str, optional
539
+ Base URL of the files management API
540
+ organization_id : str, optional
541
+ ID of organization, if using a JWT for authentication
542
+
543
+ Returns
544
+ -------
545
+ :class:`FilesManager`
546
+
547
+ """
548
+ return FilesManager(
549
+ access_token=access_token, base_url=base_url,
550
+ version=version, organization_id=organization_id,
551
+ )
552
+
553
+
554
+ class FileSpace(FileLocation):
555
+ """
556
+ FileSpace manager.
557
+
558
+ This object is not instantiated directly.
559
+ It is returned by ``FilesManager.personal_space``, ``FilesManager.shared_space``
560
+ or ``FileManger.models_space``.
561
+
562
+ """
563
+
564
+ def __init__(self, location: str, manager: FilesManager):
565
+ self._location = location
566
+ self._manager = manager
567
+
568
+ def open(
569
+ self,
570
+ path: PathLike,
571
+ mode: str = 'r',
572
+ encoding: Optional[str] = None,
573
+ ) -> Union[io.StringIO, io.BytesIO]:
574
+ """
575
+ Open a file path for reading or writing.
576
+
577
+ Parameters
578
+ ----------
579
+ path : Path or str
580
+ The file path to read / write
581
+ mode : str, optional
582
+ The read / write mode. The following modes are supported:
583
+ * 'r' open for reading (default)
584
+ * 'w' open for writing, truncating the file first
585
+ * 'x' create a new file and open it for writing
586
+ The data type can be specified by adding one of the following:
587
+ * 'b' binary mode
588
+ * 't' text mode (default)
589
+ encoding : str, optional
590
+ The string encoding to use for text
591
+
592
+ Returns
593
+ -------
594
+ FilesObjectBytesReader - 'rb' or 'b' mode
595
+ FilesObjectBytesWriter - 'wb' or 'xb' mode
596
+ FilesObjectTextReader - 'r' or 'rt' mode
597
+ FilesObjectTextWriter - 'w', 'x', 'wt' or 'xt' mode
598
+
599
+ """
600
+ if '+' in mode or 'a' in mode:
601
+ raise ManagementError(msg='modifying an existing file is not supported')
602
+
603
+ if 'w' in mode or 'x' in mode:
604
+ exists = self.exists(path)
605
+ if exists:
606
+ if 'x' in mode:
607
+ raise FileExistsError(f'file path already exists: {path}')
608
+ self.remove(path)
609
+ if 'b' in mode:
610
+ return FilesObjectBytesWriter(b'', self, path)
611
+ return FilesObjectTextWriter('', self, path)
612
+
613
+ if 'r' in mode:
614
+ content = self.download_file(path)
615
+ if isinstance(content, bytes):
616
+ if 'b' in mode:
617
+ return FilesObjectBytesReader(content)
618
+ encoding = 'utf-8' if encoding is None else encoding
619
+ return FilesObjectTextReader(content.decode(encoding))
620
+
621
+ if isinstance(content, str):
622
+ return FilesObjectTextReader(content)
623
+
624
+ raise ValueError(f'unrecognized file content type: {type(content)}')
625
+
626
+ raise ValueError(f'must have one of create/read/write mode specified: {mode}')
627
+
628
+ def upload_file(
629
+ self,
630
+ local_path: Union[PathLike, io.IOBase],
631
+ path: PathLike,
632
+ *,
633
+ overwrite: bool = False,
634
+ ) -> FilesObject:
635
+ """
636
+ Upload a local file.
637
+
638
+ Parameters
639
+ ----------
640
+ local_path : Path or str or file-like
641
+ Path to the local file or an open file object
642
+ path : Path or str
643
+ Path to the file
644
+ overwrite : bool, optional
645
+ Should the ``path`` be overwritten if it exists already?
646
+
647
+ """
648
+ if isinstance(local_path, io.IOBase):
649
+ pass
650
+ elif not os.path.isfile(local_path):
651
+ raise IsADirectoryError(f'local path is not a file: {local_path}')
652
+
653
+ if self.exists(path):
654
+ if not overwrite:
655
+ raise OSError(f'file path already exists: {path}')
656
+
657
+ self.remove(path)
658
+
659
+ if isinstance(local_path, io.IOBase):
660
+ return self._upload(local_path, path, overwrite=overwrite)
661
+
662
+ return self._upload(open(local_path, 'rb'), path, overwrite=overwrite)
663
+
664
+ def upload_folder(
665
+ self,
666
+ local_path: PathLike,
667
+ path: PathLike,
668
+ *,
669
+ overwrite: bool = False,
670
+ recursive: bool = True,
671
+ include_root: bool = False,
672
+ ignore: Optional[Union[PathLike, List[PathLike]]] = None,
673
+ ) -> FilesObject:
674
+ """
675
+ Upload a folder recursively.
676
+
677
+ Only the contents of the folder are uploaded. To include the
678
+ folder name itself in the target path use ``include_root=True``.
679
+
680
+ Parameters
681
+ ----------
682
+ local_path : Path or str
683
+ Local directory to upload
684
+ path : Path or str
685
+ Path of folder to upload to
686
+ overwrite : bool, optional
687
+ If a file already exists, should it be overwritten?
688
+ recursive : bool, optional
689
+ Should nested folders be uploaded?
690
+ include_root : bool, optional
691
+ Should the local root folder itself be uploaded as the top folder?
692
+ ignore : Path or str or List[Path] or List[str], optional
693
+ Glob patterns of files to ignore, for example, '**/*.pyc` will
694
+ ignore all '*.pyc' files in the directory tree
695
+
696
+ """
697
+ if not os.path.isdir(local_path):
698
+ raise NotADirectoryError(f'local path is not a directory: {local_path}')
699
+
700
+ if not path:
701
+ path = local_path
702
+
703
+ ignore_files = set()
704
+ if ignore:
705
+ if isinstance(ignore, list):
706
+ for item in ignore:
707
+ ignore_files.update(glob.glob(str(item), recursive=recursive))
708
+ else:
709
+ ignore_files.update(glob.glob(str(ignore), recursive=recursive))
710
+
711
+ for dir_path, _, files in os.walk(str(local_path)):
712
+ for fname in files:
713
+ if ignore_files and fname in ignore_files:
714
+ continue
715
+
716
+ local_file_path = os.path.join(dir_path, fname)
717
+ remote_path = os.path.join(
718
+ path,
719
+ local_file_path.lstrip(str(local_path)),
720
+ )
721
+ self.upload_file(
722
+ local_path=local_file_path,
723
+ path=remote_path,
724
+ overwrite=overwrite,
725
+ )
726
+ return self.info(path)
727
+
728
+ def _upload(
729
+ self,
730
+ content: Union[str, bytes, io.IOBase],
731
+ path: PathLike,
732
+ *,
733
+ overwrite: bool = False,
734
+ ) -> FilesObject:
735
+ """
736
+ Upload content to a file.
737
+
738
+ Parameters
739
+ ----------
740
+ content : str or bytes or file-like
741
+ Content to upload
742
+ path : Path or str
743
+ Path to the file
744
+ overwrite : bool, optional
745
+ Should the ``path`` be overwritten if it exists already?
746
+
747
+ """
748
+ if self.exists(path):
749
+ if not overwrite:
750
+ raise OSError(f'file path already exists: {path}')
751
+ self.remove(path)
752
+
753
+ self._manager._put(
754
+ f'files/fs/{self._location}/{path}',
755
+ files={'file': content},
756
+ headers={'Content-Type': None},
757
+ )
758
+
759
+ return self.info(path)
760
+
761
+ def mkdir(self, path: PathLike, overwrite: bool = False) -> FilesObject:
762
+ """
763
+ Make a directory in the file space.
764
+
765
+ Parameters
766
+ ----------
767
+ path : Path or str
768
+ Path of the folder to create
769
+ overwrite : bool, optional
770
+ Should the file path be overwritten if it exists already?
771
+
772
+ Returns
773
+ -------
774
+ FilesObject
775
+
776
+ """
777
+ raise ManagementError(
778
+ msg='Operation not supported: directories are currently not allowed '
779
+ 'in Files API',
780
+ )
781
+
782
+ mkdirs = mkdir
783
+
784
+ def rename(
785
+ self,
786
+ old_path: PathLike,
787
+ new_path: PathLike,
788
+ *,
789
+ overwrite: bool = False,
790
+ ) -> FilesObject:
791
+ """
792
+ Move the file to a new location.
793
+
794
+ Parameters
795
+ -----------
796
+ old_path : Path or str
797
+ Original location of the path
798
+ new_path : Path or str
799
+ New location of the path
800
+ overwrite : bool, optional
801
+ Should the ``new_path`` be overwritten if it exists already?
802
+
803
+ """
804
+ if not self.exists(old_path):
805
+ raise OSError(f'file path does not exist: {old_path}')
806
+
807
+ if str(old_path).endswith('/') or str(new_path).endswith('/'):
808
+ raise ManagementError(
809
+ msg='Operation not supported: directories are currently not allowed '
810
+ 'in Files API',
811
+ )
812
+
813
+ if self.exists(new_path):
814
+ if not overwrite:
815
+ raise OSError(f'file path already exists: {new_path}')
816
+
817
+ self.remove(new_path)
818
+
819
+ self._manager._patch(
820
+ f'files/fs/{self._location}/{old_path}',
821
+ json=dict(newPath=new_path),
822
+ )
823
+
824
+ return self.info(new_path)
825
+
826
+ def info(self, path: PathLike) -> FilesObject:
827
+ """
828
+ Return information about a file location.
829
+
830
+ Parameters
831
+ ----------
832
+ path : Path or str
833
+ Path to the file
834
+
835
+ Returns
836
+ -------
837
+ FilesObject
838
+
839
+ """
840
+ res = self._manager._get(
841
+ re.sub(r'/+$', r'/', f'files/fs/{self._location}/{path}'),
842
+ params=dict(metadata=1),
843
+ ).json()
844
+
845
+ return FilesObject.from_dict(res, self)
846
+
847
+ def exists(self, path: PathLike) -> bool:
848
+ """
849
+ Does the given file path exist?
850
+
851
+ Parameters
852
+ ----------
853
+ path : Path or str
854
+ Path to file object
855
+
856
+ Returns
857
+ -------
858
+ bool
859
+
860
+ """
861
+ try:
862
+ self.info(path)
863
+ return True
864
+ except ManagementError as exc:
865
+ if exc.errno == 404:
866
+ return False
867
+ raise
868
+
869
+ def is_dir(self, path: PathLike) -> bool:
870
+ """
871
+ Is the given file path a directory?
872
+
873
+ Parameters
874
+ ----------
875
+ path : Path or str
876
+ Path to file object
877
+
878
+ Returns
879
+ -------
880
+ bool
881
+
882
+ """
883
+ try:
884
+ return self.info(path).type == 'directory'
885
+ except ManagementError as exc:
886
+ if exc.errno == 404:
887
+ return False
888
+ raise
889
+
890
+ def is_file(self, path: PathLike) -> bool:
891
+ """
892
+ Is the given file path a file?
893
+
894
+ Parameters
895
+ ----------
896
+ path : Path or str
897
+ Path to file object
898
+
899
+ Returns
900
+ -------
901
+ bool
902
+
903
+ """
904
+ try:
905
+ return self.info(path).type != 'directory'
906
+ except ManagementError as exc:
907
+ if exc.errno == 404:
908
+ return False
909
+ raise
910
+
911
+ def _listdir(self, path: PathLike, *, recursive: bool = False) -> List[str]:
912
+ """
913
+ Return the names of files in a directory.
914
+
915
+ Parameters
916
+ ----------
917
+ path : Path or str
918
+ Path to the folder
919
+ recursive : bool, optional
920
+ Should folders be listed recursively?
921
+
922
+ """
923
+ res = self._manager._get(
924
+ f'files/fs/{self._location}/{path}',
925
+ ).json()
926
+
927
+ if recursive:
928
+ out = []
929
+ for item in res['content'] or []:
930
+ out.append(item['path'])
931
+ if item['type'] == 'directory':
932
+ out.extend(self._listdir(item['path'], recursive=recursive))
933
+ return out
934
+
935
+ return [x['path'] for x in res['content'] or []]
936
+
937
+ def listdir(
938
+ self,
939
+ path: PathLike = '/',
940
+ *,
941
+ recursive: bool = False,
942
+ ) -> List[str]:
943
+ """
944
+ List the files / folders at the given path.
945
+
946
+ Parameters
947
+ ----------
948
+ path : Path or str, optional
949
+ Path to the file location
950
+
951
+ Returns
952
+ -------
953
+ List[str]
954
+
955
+ """
956
+ path = re.sub(r'^(\./|/)+', r'', str(path))
957
+ path = re.sub(r'/+$', r'', path) + '/'
958
+
959
+ if not self.is_dir(path):
960
+ raise NotADirectoryError(f'path is not a directory: {path}')
961
+
962
+ out = self._listdir(path, recursive=recursive)
963
+ if path != '/':
964
+ path_n = len(path.split('/')) - 1
965
+ out = ['/'.join(x.split('/')[path_n:]) for x in out]
966
+ return out
967
+
968
+ def download_file(
969
+ self,
970
+ path: PathLike,
971
+ local_path: Optional[PathLike] = None,
972
+ *,
973
+ overwrite: bool = False,
974
+ encoding: Optional[str] = None,
975
+ ) -> Optional[Union[bytes, str]]:
976
+ """
977
+ Download the content of a file path.
978
+
979
+ Parameters
980
+ ----------
981
+ path : Path or str
982
+ Path to the file
983
+ local_path : Path or str
984
+ Path to local file target location
985
+ overwrite : bool, optional
986
+ Should an existing file be overwritten if it exists?
987
+ encoding : str, optional
988
+ Encoding used to convert the resulting data
989
+
990
+ Returns
991
+ -------
992
+ bytes or str - ``local_path`` is None
993
+ None - ``local_path`` is a Path or str
994
+
995
+ """
996
+ if local_path is not None and not overwrite and os.path.exists(local_path):
997
+ raise OSError('target file already exists; use overwrite=True to replace')
998
+ if self.is_dir(path):
999
+ raise IsADirectoryError(f'file path is a directory: {path}')
1000
+
1001
+ out = self._manager._get(
1002
+ f'files/fs/{self._location}/{path}',
1003
+ ).content
1004
+
1005
+ if local_path is not None:
1006
+ with open(local_path, 'wb') as outfile:
1007
+ outfile.write(out)
1008
+ return None
1009
+
1010
+ if encoding:
1011
+ return out.decode(encoding)
1012
+
1013
+ return out
1014
+
1015
+ def download_folder(
1016
+ self,
1017
+ path: PathLike,
1018
+ local_path: PathLike = '.',
1019
+ *,
1020
+ overwrite: bool = False,
1021
+ ) -> None:
1022
+ """
1023
+ Download a FileSpace folder to a local directory.
1024
+
1025
+ Parameters
1026
+ ----------
1027
+ path : Path or str
1028
+ Directory path
1029
+ local_path : Path or str
1030
+ Path to local directory target location
1031
+ overwrite : bool, optional
1032
+ Should an existing directory / files be overwritten if they exist?
1033
+
1034
+ """
1035
+
1036
+ if local_path is not None and not overwrite and os.path.exists(local_path):
1037
+ raise OSError('target path already exists; use overwrite=True to replace')
1038
+
1039
+ if not self.is_dir(path):
1040
+ raise NotADirectoryError(f'path is not a directory: {path}')
1041
+
1042
+ files = self.listdir(path, recursive=True)
1043
+ for f in files:
1044
+ remote_path = os.path.join(path, f)
1045
+ if self.is_dir(remote_path):
1046
+ continue
1047
+ target = os.path.normpath(os.path.join(local_path, f))
1048
+ os.makedirs(os.path.dirname(target), exist_ok=True)
1049
+ self.download_file(remote_path, target, overwrite=overwrite)
1050
+
1051
+ def remove(self, path: PathLike) -> None:
1052
+ """
1053
+ Delete a file location.
1054
+
1055
+ Parameters
1056
+ ----------
1057
+ path : Path or str
1058
+ Path to the location
1059
+
1060
+ """
1061
+ if self.is_dir(path):
1062
+ raise IsADirectoryError('file path is a directory')
1063
+
1064
+ self._manager._delete(f'files/fs/{self._location}/{path}')
1065
+
1066
+ def removedirs(self, path: PathLike) -> None:
1067
+ """
1068
+ Delete a folder recursively.
1069
+
1070
+ Parameters
1071
+ ----------
1072
+ path : Path or str
1073
+ Path to the file location
1074
+
1075
+ """
1076
+ if not self.is_dir(path):
1077
+ raise NotADirectoryError('path is not a directory')
1078
+
1079
+ self._manager._delete(f'files/fs/{self._location}/{path}')
1080
+
1081
+ def rmdir(self, path: PathLike) -> None:
1082
+ """
1083
+ Delete a folder.
1084
+
1085
+ Parameters
1086
+ ----------
1087
+ path : Path or str
1088
+ Path to the file location
1089
+
1090
+ """
1091
+ raise ManagementError(
1092
+ msg='Operation not supported: directories are currently not allowed '
1093
+ 'in Files API',
1094
+ )
1095
+
1096
+ def __str__(self) -> str:
1097
+ """Return string representation."""
1098
+ return vars_to_str(self)
1099
+
1100
+ def __repr__(self) -> str:
1101
+ """Return string representation."""
1102
+ return str(self)